2024-12-08 01:13:51,049 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 01:13:51,059 main DEBUG Took 0.008870 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 01:13:51,059 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 01:13:51,059 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 01:13:51,060 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 01:13:51,061 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,068 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 01:13:51,078 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,080 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,080 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,081 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,082 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,083 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,084 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,085 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,086 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,086 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:13:51,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,087 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 01:13:51,089 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:13:51,090 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 01:13:51,092 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 01:13:51,092 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 01:13:51,093 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 01:13:51,093 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 01:13:51,100 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 01:13:51,102 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 01:13:51,104 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 01:13:51,104 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 01:13:51,104 main DEBUG createAppenders(={Console}) 2024-12-08 01:13:51,105 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-08 01:13:51,105 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 01:13:51,105 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-08 01:13:51,106 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 01:13:51,106 main DEBUG OutputStream closed 2024-12-08 01:13:51,106 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 01:13:51,107 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 01:13:51,107 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-08 01:13:51,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 01:13:51,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 01:13:51,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 01:13:51,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 01:13:51,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 01:13:51,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 01:13:51,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 01:13:51,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 01:13:51,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 01:13:51,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 01:13:51,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 01:13:51,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 01:13:51,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 01:13:51,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 01:13:51,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 01:13:51,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 01:13:51,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 01:13:51,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 01:13:51,176 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 01:13:51,176 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-08 01:13:51,176 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 01:13:51,177 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-08T01:13:51,374 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8 2024-12-08 01:13:51,376 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 01:13:51,376 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T01:13:51,384 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-08T01:13:51,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T01:13:51,414 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269, deleteOnExit=true 2024-12-08T01:13:51,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T01:13:51,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/test.cache.data in system properties and HBase conf 2024-12-08T01:13:51,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T01:13:51,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/hadoop.log.dir in system properties and HBase conf 2024-12-08T01:13:51,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T01:13:51,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T01:13:51,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T01:13:51,496 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T01:13:51,581 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T01:13:51,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T01:13:51,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T01:13:51,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T01:13:51,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T01:13:51,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T01:13:51,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T01:13:51,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T01:13:51,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T01:13:51,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T01:13:51,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/nfs.dump.dir in system properties and HBase conf 2024-12-08T01:13:51,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/java.io.tmpdir in system properties and HBase conf 2024-12-08T01:13:51,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T01:13:51,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T01:13:51,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T01:13:52,680 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T01:13:52,750 INFO [Time-limited test {}] log.Log(170): Logging initialized @2269ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T01:13:52,816 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:13:52,874 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:13:52,895 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:13:52,895 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:13:52,896 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T01:13:52,908 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:13:52,911 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:13:52,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:13:53,085 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/java.io.tmpdir/jetty-localhost-38841-hadoop-hdfs-3_4_1-tests_jar-_-any-4078849077480286387/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T01:13:53,105 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38841} 2024-12-08T01:13:53,105 INFO [Time-limited test {}] server.Server(415): Started @2625ms 2024-12-08T01:13:53,664 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:13:53,671 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:13:53,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:13:53,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:13:53,672 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T01:13:53,673 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:13:53,674 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:13:53,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/java.io.tmpdir/jetty-localhost-33455-hadoop-hdfs-3_4_1-tests_jar-_-any-6518588312814143778/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:13:53,767 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33455} 2024-12-08T01:13:53,767 INFO [Time-limited test {}] server.Server(415): Started @3287ms 2024-12-08T01:13:53,811 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:13:54,742 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/data/data2/current/BP-261549399-172.17.0.2-1733620432050/current, will proceed with Du for space computation calculation, 2024-12-08T01:13:54,742 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/data/data1/current/BP-261549399-172.17.0.2-1733620432050/current, will proceed with Du for space computation calculation, 2024-12-08T01:13:54,773 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:13:54,814 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xceb5fda28d849e86 with lease ID 0xf98fa7d42c07bcdd: Processing first storage report for DS-f60031a8-7b54-4600-adad-4e43581d9284 from datanode DatanodeRegistration(127.0.0.1:40511, datanodeUuid=6c6b5ef4-4866-4f22-9083-19ac04e152bb, infoPort=37971, infoSecurePort=0, ipcPort=41393, storageInfo=lv=-57;cid=testClusterID;nsid=1471904477;c=1733620432050) 2024-12-08T01:13:54,815 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xceb5fda28d849e86 with lease ID 0xf98fa7d42c07bcdd: from storage DS-f60031a8-7b54-4600-adad-4e43581d9284 node DatanodeRegistration(127.0.0.1:40511, datanodeUuid=6c6b5ef4-4866-4f22-9083-19ac04e152bb, infoPort=37971, infoSecurePort=0, ipcPort=41393, storageInfo=lv=-57;cid=testClusterID;nsid=1471904477;c=1733620432050), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T01:13:54,815 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xceb5fda28d849e86 with lease ID 0xf98fa7d42c07bcdd: Processing first storage report for DS-abd6dd98-c33d-46c7-95a0-38db6ca8c083 from datanode DatanodeRegistration(127.0.0.1:40511, datanodeUuid=6c6b5ef4-4866-4f22-9083-19ac04e152bb, infoPort=37971, infoSecurePort=0, ipcPort=41393, storageInfo=lv=-57;cid=testClusterID;nsid=1471904477;c=1733620432050) 2024-12-08T01:13:54,815 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xceb5fda28d849e86 with lease ID 0xf98fa7d42c07bcdd: from storage DS-abd6dd98-c33d-46c7-95a0-38db6ca8c083 node DatanodeRegistration(127.0.0.1:40511, datanodeUuid=6c6b5ef4-4866-4f22-9083-19ac04e152bb, infoPort=37971, infoSecurePort=0, ipcPort=41393, storageInfo=lv=-57;cid=testClusterID;nsid=1471904477;c=1733620432050), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:13:54,914 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8 2024-12-08T01:13:54,996 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/zookeeper_0, clientPort=63353, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T01:13:55,005 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63353 2024-12-08T01:13:55,013 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:55,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:55,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:13:55,620 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 with version=8 2024-12-08T01:13:55,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/hbase-staging 2024-12-08T01:13:55,699 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T01:13:55,922 INFO [Time-limited test {}] client.ConnectionUtils(128): master/96982154c400:0 server-side Connection retries=45 2024-12-08T01:13:55,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:13:55,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:13:55,936 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:13:55,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:13:55,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:13:56,047 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T01:13:56,094 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T01:13:56,104 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T01:13:56,108 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:13:56,129 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4652 (auto-detected) 2024-12-08T01:13:56,130 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T01:13:56,146 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37013 2024-12-08T01:13:56,166 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37013 connecting to ZooKeeper ensemble=127.0.0.1:63353 2024-12-08T01:13:56,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:370130x0, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:13:56,353 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37013-0x10002fc8ee80000 connected 2024-12-08T01:13:56,514 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:56,518 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:56,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:13:56,532 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720, hbase.cluster.distributed=false 2024-12-08T01:13:56,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:13:56,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37013 2024-12-08T01:13:56,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37013 2024-12-08T01:13:56,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37013 2024-12-08T01:13:56,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37013 2024-12-08T01:13:56,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37013 2024-12-08T01:13:56,662 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:13:56,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:13:56,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:13:56,664 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:13:56,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:13:56,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:13:56,667 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:13:56,669 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:13:56,670 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44843 2024-12-08T01:13:56,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44843 connecting to ZooKeeper ensemble=127.0.0.1:63353 2024-12-08T01:13:56,672 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:56,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:56,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448430x0, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:13:56,690 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:448430x0, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:13:56,690 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44843-0x10002fc8ee80001 connected 2024-12-08T01:13:56,693 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:13:56,700 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:13:56,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:13:56,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:13:56,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44843 2024-12-08T01:13:56,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44843 2024-12-08T01:13:56,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44843 2024-12-08T01:13:56,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44843 2024-12-08T01:13:56,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44843 2024-12-08T01:13:56,722 DEBUG [M:0;96982154c400:37013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;96982154c400:37013 2024-12-08T01:13:56,723 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/96982154c400,37013,1733620435769 2024-12-08T01:13:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:13:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:13:56,743 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/96982154c400,37013,1733620435769 2024-12-08T01:13:56,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:13:56,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:56,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:56,775 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T01:13:56,776 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/96982154c400,37013,1733620435769 from backup master directory 2024-12-08T01:13:56,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/96982154c400,37013,1733620435769 2024-12-08T01:13:56,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:13:56,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:13:56,784 WARN [master/96982154c400:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:13:56,784 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=96982154c400,37013,1733620435769 2024-12-08T01:13:56,786 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T01:13:56,787 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T01:13:56,836 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/hbase.id] with ID: ee334ca5-882c-4f4a-9655-afec0198aa70 2024-12-08T01:13:56,836 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/.tmp/hbase.id 2024-12-08T01:13:56,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:13:57,248 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/.tmp/hbase.id]:[hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/hbase.id] 2024-12-08T01:13:57,300 INFO [master/96982154c400:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:13:57,305 INFO [master/96982154c400:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T01:13:57,321 INFO [master/96982154c400:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-08T01:13:57,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:57,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:57,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:13:57,803 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:13:57,805 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T01:13:57,819 DEBUG [master/96982154c400:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:13:57,823 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:13:57,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:13:58,274 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store 2024-12-08T01:13:58,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:13:58,703 INFO [master/96982154c400:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T01:13:58,706 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:13:58,708 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T01:13:58,708 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:13:58,708 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:13:58,710 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T01:13:58,710 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:13:58,710 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:13:58,711 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733620438708Disabling compacts and flushes for region at 1733620438708Disabling writes for close at 1733620438710 (+2 ms)Writing region close event to WAL at 1733620438710Closed at 1733620438710 2024-12-08T01:13:58,714 WARN [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/.initializing 2024-12-08T01:13:58,714 DEBUG [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/WALs/96982154c400,37013,1733620435769 2024-12-08T01:13:58,722 INFO [master/96982154c400:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:13:58,737 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C37013%2C1733620435769, suffix=, logDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/WALs/96982154c400,37013,1733620435769, archiveDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/oldWALs, maxLogs=10 2024-12-08T01:13:58,757 DEBUG [master/96982154c400:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/WALs/96982154c400,37013,1733620435769/96982154c400%2C37013%2C1733620435769.1733620438741, exclude list is [], retry=0 2024-12-08T01:13:58,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40511,DS-f60031a8-7b54-4600-adad-4e43581d9284,DISK] 2024-12-08T01:13:58,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T01:13:58,804 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/WALs/96982154c400,37013,1733620435769/96982154c400%2C37013%2C1733620435769.1733620438741 2024-12-08T01:13:58,805 DEBUG [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37971:37971)] 2024-12-08T01:13:58,806 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:13:58,806 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:13:58,809 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,810 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T01:13:58,865 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:13:58,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:13:58,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T01:13:58,871 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:13:58,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:13:58,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T01:13:58,875 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:13:58,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:13:58,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T01:13:58,878 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:13:58,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:13:58,879 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,883 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,884 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,890 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,891 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,895 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:13:58,899 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:13:58,903 DEBUG [master/96982154c400:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:13:58,905 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68994011, jitterRate=0.028090879321098328}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:13:58,909 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733620438820Initializing all the Stores at 1733620438823 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620438823Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620438824 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620438824Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620438824Cleaning up temporary data from old regions at 1733620438891 (+67 ms)Region opened successfully at 1733620438909 (+18 ms) 2024-12-08T01:13:58,911 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T01:13:58,940 DEBUG [master/96982154c400:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcc19d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:13:58,965 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T01:13:58,975 INFO [master/96982154c400:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T01:13:58,975 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T01:13:58,977 INFO [master/96982154c400:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T01:13:58,978 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T01:13:58,982 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-08T01:13:58,982 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T01:13:59,004 INFO [master/96982154c400:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T01:13:59,012 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T01:13:59,057 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T01:13:59,061 INFO [master/96982154c400:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T01:13:59,064 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T01:13:59,076 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T01:13:59,079 INFO [master/96982154c400:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T01:13:59,082 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T01:13:59,088 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T01:13:59,090 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T01:13:59,099 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T01:13:59,120 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T01:13:59,130 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T01:13:59,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:13:59,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:13:59,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:59,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:59,145 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=96982154c400,37013,1733620435769, sessionid=0x10002fc8ee80000, setting cluster-up flag (Was=false) 2024-12-08T01:13:59,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:59,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:59,204 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T01:13:59,207 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=96982154c400,37013,1733620435769 2024-12-08T01:13:59,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:59,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:13:59,257 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T01:13:59,261 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=96982154c400,37013,1733620435769 2024-12-08T01:13:59,271 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T01:13:59,315 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(746): ClusterId : ee334ca5-882c-4f4a-9655-afec0198aa70 2024-12-08T01:13:59,317 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:13:59,333 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:13:59,333 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:13:59,335 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T01:13:59,343 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:13:59,343 DEBUG [RS:0;96982154c400:44843 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763e7329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:13:59,344 INFO [master/96982154c400:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T01:13:59,350 INFO [master/96982154c400:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T01:13:59,355 DEBUG [RS:0;96982154c400:44843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;96982154c400:44843 2024-12-08T01:13:59,355 DEBUG [master/96982154c400:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 96982154c400,37013,1733620435769 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T01:13:59,358 INFO [RS:0;96982154c400:44843 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:13:59,358 INFO [RS:0;96982154c400:44843 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:13:59,358 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:13:59,361 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,37013,1733620435769 with port=44843, startcode=1733620436630 2024-12-08T01:13:59,361 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:13:59,361 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:13:59,361 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:13:59,361 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:13:59,362 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/96982154c400:0, corePoolSize=10, maxPoolSize=10 2024-12-08T01:13:59,362 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,362 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:13:59,362 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,363 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733620469363 2024-12-08T01:13:59,365 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T01:13:59,366 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T01:13:59,367 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:13:59,368 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T01:13:59,370 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T01:13:59,370 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T01:13:59,370 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T01:13:59,370 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T01:13:59,371 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,373 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:13:59,373 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T01:13:59,374 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T01:13:59,375 DEBUG [RS:0;96982154c400:44843 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:13:59,376 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T01:13:59,376 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T01:13:59,381 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T01:13:59,381 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T01:13:59,383 DEBUG [master/96982154c400:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620439382,5,FailOnTimeoutGroup] 2024-12-08T01:13:59,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:13:59,385 DEBUG [master/96982154c400:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620439383,5,FailOnTimeoutGroup] 2024-12-08T01:13:59,385 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,385 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T01:13:59,386 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,386 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,410 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36229, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:13:59,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37013 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,44843,1733620436630 2024-12-08T01:13:59,419 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37013 {}] master.ServerManager(517): Registering regionserver=96982154c400,44843,1733620436630 2024-12-08T01:13:59,432 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:13:59,432 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41355 2024-12-08T01:13:59,432 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:13:59,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:13:59,447 DEBUG [RS:0;96982154c400:44843 {}] zookeeper.ZKUtil(111): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,44843,1733620436630 2024-12-08T01:13:59,448 WARN [RS:0;96982154c400:44843 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:13:59,448 INFO [RS:0;96982154c400:44843 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:13:59,448 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630 2024-12-08T01:13:59,450 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,44843,1733620436630] 2024-12-08T01:13:59,471 INFO [RS:0;96982154c400:44843 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:13:59,482 INFO [RS:0;96982154c400:44843 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:13:59,486 INFO [RS:0;96982154c400:44843 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:13:59,487 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,487 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:13:59,492 INFO [RS:0;96982154c400:44843 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:13:59,493 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,493 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,493 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,494 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,495 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,495 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,495 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:13:59,495 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:13:59,495 DEBUG [RS:0;96982154c400:44843 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:13:59,496 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,496 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,496 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,496 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,496 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,496 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,44843,1733620436630-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:13:59,513 INFO [RS:0;96982154c400:44843 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:13:59,515 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,44843,1733620436630-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,515 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,515 INFO [RS:0;96982154c400:44843 {}] regionserver.Replication(171): 96982154c400,44843,1733620436630 started 2024-12-08T01:13:59,535 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:13:59,536 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,44843,1733620436630, RpcServer on 96982154c400/172.17.0.2:44843, sessionid=0x10002fc8ee80001 2024-12-08T01:13:59,537 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:13:59,537 DEBUG [RS:0;96982154c400:44843 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,44843,1733620436630 2024-12-08T01:13:59,537 DEBUG [RS:0;96982154c400:44843 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,44843,1733620436630' 2024-12-08T01:13:59,537 DEBUG [RS:0;96982154c400:44843 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:13:59,538 DEBUG [RS:0;96982154c400:44843 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:13:59,539 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:13:59,539 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:13:59,539 DEBUG [RS:0;96982154c400:44843 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,44843,1733620436630 2024-12-08T01:13:59,539 DEBUG [RS:0;96982154c400:44843 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,44843,1733620436630' 2024-12-08T01:13:59,539 DEBUG [RS:0;96982154c400:44843 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:13:59,540 DEBUG [RS:0;96982154c400:44843 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:13:59,540 DEBUG [RS:0;96982154c400:44843 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:13:59,541 INFO [RS:0;96982154c400:44843 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:13:59,541 INFO [RS:0;96982154c400:44843 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:13:59,647 INFO [RS:0;96982154c400:44843 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:13:59,652 INFO [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C44843%2C1733620436630, suffix=, logDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630, archiveDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/oldWALs, maxLogs=32 2024-12-08T01:13:59,668 DEBUG [RS:0;96982154c400:44843 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630/96982154c400%2C44843%2C1733620436630.1733620439656, exclude list is [], retry=0 2024-12-08T01:13:59,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40511,DS-f60031a8-7b54-4600-adad-4e43581d9284,DISK] 2024-12-08T01:13:59,675 INFO [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630/96982154c400%2C44843%2C1733620436630.1733620439656 2024-12-08T01:13:59,675 DEBUG [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37971:37971)] 2024-12-08T01:13:59,789 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T01:13:59,791 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:13:59,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741833_1009 (size=32) 2024-12-08T01:14:00,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:00,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:14:00,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:14:00,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:14:00,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:14:00,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:14:00,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:14:00,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:14:00,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:14:00,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:14:00,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740 2024-12-08T01:14:00,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740 2024-12-08T01:14:00,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:14:00,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:14:00,230 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:14:00,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:14:00,236 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:14:00,236 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58864834, jitterRate=-0.1228456199169159}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:14:00,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733620440204Initializing all the Stores at 1733620440207 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620440208 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620440208Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620440208Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620440208Cleaning up temporary data from old regions at 1733620440230 (+22 ms)Region opened successfully at 1733620440238 (+8 ms) 2024-12-08T01:14:00,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:14:00,238 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:14:00,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:14:00,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:14:00,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:14:00,239 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:14:00,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620440238Disabling compacts and flushes for region at 1733620440238Disabling writes for close at 1733620440238Writing region close event to WAL at 1733620440239 (+1 ms)Closed at 1733620440239 2024-12-08T01:14:00,242 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:14:00,242 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T01:14:00,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T01:14:00,254 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:14:00,256 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T01:14:00,413 DEBUG [96982154c400:37013 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T01:14:00,425 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:00,430 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,44843,1733620436630, state=OPENING 2024-12-08T01:14:00,467 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T01:14:00,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:14:00,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:14:00,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:14:00,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:14:00,483 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:14:00,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,44843,1733620436630}] 2024-12-08T01:14:00,664 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:14:00,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34077, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:14:00,677 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T01:14:00,677 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:14:00,678 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T01:14:00,681 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C44843%2C1733620436630.meta, suffix=.meta, logDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630, archiveDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/oldWALs, maxLogs=32 2024-12-08T01:14:00,694 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630/96982154c400%2C44843%2C1733620436630.meta.1733620440682.meta, exclude list is [], retry=0 2024-12-08T01:14:00,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40511,DS-f60031a8-7b54-4600-adad-4e43581d9284,DISK] 2024-12-08T01:14:00,700 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/WALs/96982154c400,44843,1733620436630/96982154c400%2C44843%2C1733620436630.meta.1733620440682.meta 2024-12-08T01:14:00,700 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37971:37971)] 2024-12-08T01:14:00,700 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:14:00,702 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T01:14:00,704 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T01:14:00,708 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T01:14:00,711 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T01:14:00,712 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:00,712 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T01:14:00,712 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T01:14:00,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:14:00,716 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:14:00,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:14:00,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:14:00,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:14:00,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:14:00,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:14:00,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:14:00,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:00,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:14:00,726 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:14:00,728 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740 2024-12-08T01:14:00,730 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740 2024-12-08T01:14:00,733 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:14:00,733 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:14:00,734 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:14:00,736 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:14:00,738 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63087156, jitterRate=-0.05992811918258667}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:14:00,738 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T01:14:00,739 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733620440713Writing region info on filesystem at 1733620440713Initializing all the Stores at 1733620440714 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620440714Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620440715 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620440715Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620440715Cleaning up temporary data from old regions at 1733620440733 (+18 ms)Running coprocessor post-open hooks at 1733620440738 (+5 ms)Region opened successfully at 1733620440739 (+1 ms) 2024-12-08T01:14:00,745 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733620440654 2024-12-08T01:14:00,756 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T01:14:00,756 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T01:14:00,757 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:00,759 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,44843,1733620436630, state=OPEN 2024-12-08T01:14:00,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:14:00,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:14:00,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:14:00,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:14:00,812 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=96982154c400,44843,1733620436630 2024-12-08T01:14:00,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T01:14:00,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,44843,1733620436630 in 325 msec 2024-12-08T01:14:00,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T01:14:00,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 576 msec 2024-12-08T01:14:00,832 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:14:00,832 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T01:14:00,850 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:00,851 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:00,868 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:00,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38267, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:00,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5890 sec 2024-12-08T01:14:00,889 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733620440889, completionTime=-1 2024-12-08T01:14:00,891 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T01:14:00,892 DEBUG [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T01:14:00,913 INFO [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T01:14:00,913 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733620500913 2024-12-08T01:14:00,913 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733620560913 2024-12-08T01:14:00,914 INFO [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-08T01:14:00,916 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37013,1733620435769-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:00,916 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37013,1733620435769-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:00,916 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37013,1733620435769-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:00,917 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-96982154c400:37013, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:00,918 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:00,918 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:00,924 DEBUG [master/96982154c400:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T01:14:00,941 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.156sec 2024-12-08T01:14:00,942 INFO [master/96982154c400:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T01:14:00,943 INFO [master/96982154c400:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T01:14:00,944 INFO [master/96982154c400:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T01:14:00,945 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T01:14:00,945 INFO [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T01:14:00,945 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37013,1733620435769-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:14:00,946 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37013,1733620435769-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T01:14:00,953 DEBUG [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T01:14:00,953 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T01:14:00,954 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37013,1733620435769-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:14:01,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:01,033 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T01:14:01,033 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T01:14:01,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:01,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:01,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:01,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:01,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:01,058 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:01,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:01,085 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=96982154c400,37013,1733620435769 2024-12-08T01:14:01,097 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=10212 2024-12-08T01:14:01,105 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:14:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:14:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-12-08T01:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:14:01,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:14:01,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:14:01,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:14:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:01,159 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:14:01,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-08T01:14:01,162 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:01,165 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:14:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:14:01,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741835_1011 (size=1061) 2024-12-08T01:14:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:14:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:14:01,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:14:01,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741836_1012 (size=53) 2024-12-08T01:14:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:14:01,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:01,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 13ab931b3a97ca4195bbab8a55597446, disabling compactions & flushes 2024-12-08T01:14:01,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:01,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:01,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. after waiting 0 ms 2024-12-08T01:14:01,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:01,996 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:01,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13ab931b3a97ca4195bbab8a55597446: Waiting for close lock at 1733620441995Disabling compacts and flushes for region at 1733620441995Disabling writes for close at 1733620441996 (+1 ms)Writing region close event to WAL at 1733620441996Closed at 1733620441996 2024-12-08T01:14:02,001 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:14:02,005 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733620442001"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620442001"}]},"ts":"1733620442001"} 2024-12-08T01:14:02,008 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:14:02,010 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:14:02,012 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620442010"}]},"ts":"1733620442010"} 2024-12-08T01:14:02,017 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T01:14:02,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, ASSIGN}] 2024-12-08T01:14:02,021 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, ASSIGN 2024-12-08T01:14:02,023 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, ASSIGN; state=OFFLINE, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=false 2024-12-08T01:14:02,176 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13ab931b3a97ca4195bbab8a55597446, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:02,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, ASSIGN because future has completed 2024-12-08T01:14:02,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630}] 2024-12-08T01:14:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:14:02,349 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:02,349 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:14:02,350 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,350 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:02,350 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,350 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,353 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,357 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:02,358 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13ab931b3a97ca4195bbab8a55597446 columnFamilyName A 2024-12-08T01:14:02,358 DEBUG [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:02,359 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.HStore(327): Store=13ab931b3a97ca4195bbab8a55597446/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:02,360 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,362 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:02,362 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13ab931b3a97ca4195bbab8a55597446 columnFamilyName B 2024-12-08T01:14:02,362 DEBUG [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:02,363 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.HStore(327): Store=13ab931b3a97ca4195bbab8a55597446/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:02,363 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,365 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:02,366 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13ab931b3a97ca4195bbab8a55597446 columnFamilyName C 2024-12-08T01:14:02,366 DEBUG [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:02,367 INFO [StoreOpener-13ab931b3a97ca4195bbab8a55597446-1 {}] regionserver.HStore(327): Store=13ab931b3a97ca4195bbab8a55597446/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:02,367 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:02,368 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,369 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,370 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,372 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,372 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,373 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:14:02,375 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,379 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:14:02,380 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 13ab931b3a97ca4195bbab8a55597446; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66849971, jitterRate=-0.0038578063249588013}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:14:02,380 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:02,381 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 13ab931b3a97ca4195bbab8a55597446: Running coprocessor pre-open hook at 1733620442350Writing region info on filesystem at 1733620442350Initializing all the Stores at 1733620442353 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620442353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620442353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620442353Cleaning up temporary data from old regions at 1733620442372 (+19 ms)Running coprocessor post-open hooks at 1733620442380 (+8 ms)Region opened successfully at 1733620442381 (+1 ms) 2024-12-08T01:14:02,383 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., pid=6, masterSystemTime=1733620442340 2024-12-08T01:14:02,386 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:02,386 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:02,387 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13ab931b3a97ca4195bbab8a55597446, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:02,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:14:02,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T01:14:02,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 in 207 msec 2024-12-08T01:14:02,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T01:14:02,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, ASSIGN in 377 msec 2024-12-08T01:14:02,401 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:14:02,402 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620442401"}]},"ts":"1733620442401"} 2024-12-08T01:14:02,405 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T01:14:02,406 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:14:02,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2590 sec 2024-12-08T01:14:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:14:03,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:03,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,327 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68edb023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb2f0c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,331 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,337 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:14:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:14:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f86aa4f 2024-12-08T01:14:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:14:03,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:14:03,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,348 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,352 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,355 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,358 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,362 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cae7c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,368 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7747469c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,371 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c21631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,374 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b21b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,385 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4707cca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,388 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4934d870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,391 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b19c399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,394 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a18c602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:03,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:03,397 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159e37c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:03,399 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ad5b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:03,404 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,404 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,404 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,404 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,404 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,405 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,406 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,406 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,407 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:03,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:03,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:14:03,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-08T01:14:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T01:14:03,443 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:03,446 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:03,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:03,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:14:03,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:03,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:03,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:03,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:03,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:03,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T01:14:03,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620503583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620503584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620503593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620503593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620503608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T01:14:03,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:03,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:03,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/746f05f7e3ac474f9ed75293dde7ac74 is 50, key is test_row_1/A:col10/1733620443493/Put/seqid=0 2024-12-08T01:14:03,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741837_1013 (size=9657) 2024-12-08T01:14:03,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/746f05f7e3ac474f9ed75293dde7ac74 2024-12-08T01:14:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T01:14:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620503764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620503765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620503772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620503773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620503773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/72a628dd799c4ff6931aff16f3f7c31e is 50, key is test_row_1/B:col10/1733620443493/Put/seqid=0 2024-12-08T01:14:03,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T01:14:03,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:03,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741838_1014 (size=9657) 2024-12-08T01:14:03,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/72a628dd799c4ff6931aff16f3f7c31e 2024-12-08T01:14:03,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/6f32a4cdf941409390e32693350c3be2 is 50, key is test_row_1/C:col10/1733620443493/Put/seqid=0 2024-12-08T01:14:03,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741839_1015 (size=9657) 2024-12-08T01:14:03,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/6f32a4cdf941409390e32693350c3be2 2024-12-08T01:14:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/746f05f7e3ac474f9ed75293dde7ac74 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/746f05f7e3ac474f9ed75293dde7ac74 2024-12-08T01:14:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620503973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/746f05f7e3ac474f9ed75293dde7ac74, entries=100, sequenceid=15, filesize=9.4 K 2024-12-08T01:14:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620503975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:03,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T01:14:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:03,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/72a628dd799c4ff6931aff16f3f7c31e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/72a628dd799c4ff6931aff16f3f7c31e 2024-12-08T01:14:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:03,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620503984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/72a628dd799c4ff6931aff16f3f7c31e, entries=100, sequenceid=15, filesize=9.4 K 2024-12-08T01:14:04,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/6f32a4cdf941409390e32693350c3be2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6f32a4cdf941409390e32693350c3be2 2024-12-08T01:14:04,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620503993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620503993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6f32a4cdf941409390e32693350c3be2, entries=100, sequenceid=15, filesize=9.4 K 2024-12-08T01:14:04,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 13ab931b3a97ca4195bbab8a55597446 in 539ms, sequenceid=15, compaction requested=false 2024-12-08T01:14:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T01:14:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T01:14:04,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T01:14:04,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:04,145 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:14:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:04,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:04,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a8d8769aaac1450bb6df6308919993df is 50, key is test_row_0/A:col10/1733620443573/Put/seqid=0 2024-12-08T01:14:04,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741840_1016 (size=12001) 2024-12-08T01:14:04,198 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a8d8769aaac1450bb6df6308919993df 2024-12-08T01:14:04,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/656d3c687609497a9e86084fb7e631bd is 50, key is test_row_0/B:col10/1733620443573/Put/seqid=0 2024-12-08T01:14:04,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741841_1017 (size=12001) 2024-12-08T01:14:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:04,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620504325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620504323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620504328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620504330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620504333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620504442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620504442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620504454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T01:14:04,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620504653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620504652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620504662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,668 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/656d3c687609497a9e86084fb7e631bd 2024-12-08T01:14:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3c9408d572ee41279974933750791fdd is 50, key is test_row_0/C:col10/1733620443573/Put/seqid=0 2024-12-08T01:14:04,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741842_1018 (size=12001) 2024-12-08T01:14:04,719 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3c9408d572ee41279974933750791fdd 2024-12-08T01:14:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a8d8769aaac1450bb6df6308919993df as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a8d8769aaac1450bb6df6308919993df 2024-12-08T01:14:04,757 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a8d8769aaac1450bb6df6308919993df, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T01:14:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/656d3c687609497a9e86084fb7e631bd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/656d3c687609497a9e86084fb7e631bd 2024-12-08T01:14:04,776 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/656d3c687609497a9e86084fb7e631bd, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T01:14:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3c9408d572ee41279974933750791fdd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3c9408d572ee41279974933750791fdd 2024-12-08T01:14:04,793 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3c9408d572ee41279974933750791fdd, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T01:14:04,796 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13ab931b3a97ca4195bbab8a55597446 in 651ms, sequenceid=33, compaction requested=false 2024-12-08T01:14:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-08T01:14:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-08T01:14:04,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T01:14:04,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3520 sec 2024-12-08T01:14:04,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3760 sec 2024-12-08T01:14:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:04,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:14:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:04,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a1a9db5f9e494238a91fa3c0f4ddcbcc is 50, key is test_row_0/A:col10/1733620444850/Put/seqid=0 2024-12-08T01:14:04,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741843_1019 (size=16681) 2024-12-08T01:14:04,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a1a9db5f9e494238a91fa3c0f4ddcbcc 2024-12-08T01:14:04,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/f75a77ad298b43bca9585a0c9823ecc6 is 50, key is test_row_0/B:col10/1733620444850/Put/seqid=0 2024-12-08T01:14:04,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741844_1020 (size=12001) 2024-12-08T01:14:04,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/f75a77ad298b43bca9585a0c9823ecc6 2024-12-08T01:14:04,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620504946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620504949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620504962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/0cfc5543e02b470c8abf22c3810942e7 is 50, key is test_row_0/C:col10/1733620444850/Put/seqid=0 2024-12-08T01:14:04,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620504973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620504973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:04,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741845_1021 (size=12001) 2024-12-08T01:14:04,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/0cfc5543e02b470c8abf22c3810942e7 2024-12-08T01:14:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a1a9db5f9e494238a91fa3c0f4ddcbcc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a1a9db5f9e494238a91fa3c0f4ddcbcc 2024-12-08T01:14:05,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a1a9db5f9e494238a91fa3c0f4ddcbcc, entries=250, sequenceid=49, filesize=16.3 K 2024-12-08T01:14:05,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/f75a77ad298b43bca9585a0c9823ecc6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/f75a77ad298b43bca9585a0c9823ecc6 2024-12-08T01:14:05,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/f75a77ad298b43bca9585a0c9823ecc6, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T01:14:05,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/0cfc5543e02b470c8abf22c3810942e7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0cfc5543e02b470c8abf22c3810942e7 2024-12-08T01:14:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620505063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620505063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0cfc5543e02b470c8abf22c3810942e7, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T01:14:05,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 13ab931b3a97ca4195bbab8a55597446 in 213ms, sequenceid=49, compaction requested=true 2024-12-08T01:14:05,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:05,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:05,076 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:05,076 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:05,080 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:05,082 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:05,082 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:05,082 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:05,083 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,083 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,083 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/72a628dd799c4ff6931aff16f3f7c31e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/656d3c687609497a9e86084fb7e631bd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/f75a77ad298b43bca9585a0c9823ecc6] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=32.9 K 2024-12-08T01:14:05,083 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/746f05f7e3ac474f9ed75293dde7ac74, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a8d8769aaac1450bb6df6308919993df, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a1a9db5f9e494238a91fa3c0f4ddcbcc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.4 K 2024-12-08T01:14:05,085 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a628dd799c4ff6931aff16f3f7c31e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733620443485 2024-12-08T01:14:05,086 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 656d3c687609497a9e86084fb7e631bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733620443573 2024-12-08T01:14:05,087 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746f05f7e3ac474f9ed75293dde7ac74, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733620443485 2024-12-08T01:14:05,088 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f75a77ad298b43bca9585a0c9823ecc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733620444324 2024-12-08T01:14:05,088 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d8769aaac1450bb6df6308919993df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733620443573 2024-12-08T01:14:05,089 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1a9db5f9e494238a91fa3c0f4ddcbcc, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733620444317 2024-12-08T01:14:05,139 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:05,140 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/d4c8f44ba1c24380ba4b09cee58d72ad is 50, key is test_row_0/A:col10/1733620444850/Put/seqid=0 2024-12-08T01:14:05,144 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:05,145 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/91c131c7a3ac4417a1f7d15957f6b34a is 50, key is test_row_0/B:col10/1733620444850/Put/seqid=0 2024-12-08T01:14:05,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741847_1023 (size=12104) 2024-12-08T01:14:05,168 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/91c131c7a3ac4417a1f7d15957f6b34a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/91c131c7a3ac4417a1f7d15957f6b34a 2024-12-08T01:14:05,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741846_1022 (size=12104) 2024-12-08T01:14:05,191 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 91c131c7a3ac4417a1f7d15957f6b34a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:05,192 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:05,195 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620445075; duration=0sec 2024-12-08T01:14:05,195 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:05,195 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:05,196 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:05,199 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:05,199 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:05,199 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,200 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6f32a4cdf941409390e32693350c3be2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3c9408d572ee41279974933750791fdd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0cfc5543e02b470c8abf22c3810942e7] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=32.9 K 2024-12-08T01:14:05,201 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f32a4cdf941409390e32693350c3be2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733620443485 2024-12-08T01:14:05,202 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9408d572ee41279974933750791fdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733620443573 2024-12-08T01:14:05,202 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cfc5543e02b470c8abf22c3810942e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733620444324 2024-12-08T01:14:05,238 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#11 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:05,238 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/6506dfaab68d4ade80895a9d07e9bdd0 is 50, key is test_row_0/C:col10/1733620444850/Put/seqid=0 2024-12-08T01:14:05,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741848_1024 (size=12104) 2024-12-08T01:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:05,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:14:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:05,290 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/6506dfaab68d4ade80895a9d07e9bdd0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6506dfaab68d4ade80895a9d07e9bdd0 2024-12-08T01:14:05,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/518663db9e144324b1738651337f481e is 50, key is test_row_0/A:col10/1733620445277/Put/seqid=0 2024-12-08T01:14:05,319 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 6506dfaab68d4ade80895a9d07e9bdd0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:05,319 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:05,320 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620445075; duration=0sec 2024-12-08T01:14:05,320 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:05,320 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:05,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741849_1025 (size=12001) 2024-12-08T01:14:05,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/518663db9e144324b1738651337f481e 2024-12-08T01:14:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620505355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/0babee2f88cd4eabaf4653ab4340098e is 50, key is test_row_0/B:col10/1733620445277/Put/seqid=0 2024-12-08T01:14:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620505361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741850_1026 (size=12001) 2024-12-08T01:14:05,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/0babee2f88cd4eabaf4653ab4340098e 2024-12-08T01:14:05,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3419f2b535334576b7f72e39ce5d5a58 is 50, key is test_row_0/C:col10/1733620445277/Put/seqid=0 2024-12-08T01:14:05,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741851_1027 (size=12001) 2024-12-08T01:14:05,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3419f2b535334576b7f72e39ce5d5a58 2024-12-08T01:14:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/518663db9e144324b1738651337f481e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/518663db9e144324b1738651337f481e 2024-12-08T01:14:05,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/518663db9e144324b1738651337f481e, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T01:14:05,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,467 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T01:14:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620505463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/0babee2f88cd4eabaf4653ab4340098e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0babee2f88cd4eabaf4653ab4340098e 2024-12-08T01:14:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620505473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620505473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0babee2f88cd4eabaf4653ab4340098e, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T01:14:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3419f2b535334576b7f72e39ce5d5a58 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3419f2b535334576b7f72e39ce5d5a58 2024-12-08T01:14:05,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620505492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3419f2b535334576b7f72e39ce5d5a58, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T01:14:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-08T01:14:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620505493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13ab931b3a97ca4195bbab8a55597446 in 221ms, sequenceid=68, compaction requested=false 2024-12-08T01:14:05,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T01:14:05,584 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:05,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-08T01:14:05,589 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T01:14:05,591 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:05,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:05,594 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/d4c8f44ba1c24380ba4b09cee58d72ad as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/d4c8f44ba1c24380ba4b09cee58d72ad 2024-12-08T01:14:05,611 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into d4c8f44ba1c24380ba4b09cee58d72ad(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:05,611 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:05,611 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620445072; duration=0sec 2024-12-08T01:14:05,612 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:05,612 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:05,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:05,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2164eaf1d4344f7cb2f07a801f084303 is 50, key is test_row_0/A:col10/1733620445356/Put/seqid=0 2024-12-08T01:14:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T01:14:05,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741852_1028 (size=14341) 2024-12-08T01:14:05,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:05,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620505757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620505760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620505873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620505873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:05,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:05,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:05,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T01:14:06,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620506083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620506083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2164eaf1d4344f7cb2f07a801f084303 2024-12-08T01:14:06,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5472b08cd7fa4b18be15c63d8fdc6c43 is 50, key is test_row_0/B:col10/1733620445356/Put/seqid=0 2024-12-08T01:14:06,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741853_1029 (size=12001) 2024-12-08T01:14:06,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5472b08cd7fa4b18be15c63d8fdc6c43 2024-12-08T01:14:06,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/f6e466e6f4a94225be8a0ed80bb74622 is 50, key is test_row_0/C:col10/1733620445356/Put/seqid=0 2024-12-08T01:14:06,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741854_1030 (size=12001) 2024-12-08T01:14:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T01:14:06,226 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:14:06,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,289 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T01:14:06,289 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T01:14:06,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:14:06,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T01:14:06,293 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T01:14:06,293 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T01:14:06,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T01:14:06,294 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T01:14:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620506392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620506392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620506492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,498 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620506512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620506512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,516 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,518 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:06,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/f6e466e6f4a94225be8a0ed80bb74622 2024-12-08T01:14:06,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2164eaf1d4344f7cb2f07a801f084303 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2164eaf1d4344f7cb2f07a801f084303 2024-12-08T01:14:06,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2164eaf1d4344f7cb2f07a801f084303, entries=200, sequenceid=85, filesize=14.0 K 2024-12-08T01:14:06,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5472b08cd7fa4b18be15c63d8fdc6c43 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5472b08cd7fa4b18be15c63d8fdc6c43 2024-12-08T01:14:06,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5472b08cd7fa4b18be15c63d8fdc6c43, entries=150, sequenceid=85, filesize=11.7 K 2024-12-08T01:14:06,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/f6e466e6f4a94225be8a0ed80bb74622 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/f6e466e6f4a94225be8a0ed80bb74622 2024-12-08T01:14:06,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/f6e466e6f4a94225be8a0ed80bb74622, entries=150, sequenceid=85, filesize=11.7 K 2024-12-08T01:14:06,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 13ab931b3a97ca4195bbab8a55597446 in 997ms, sequenceid=85, compaction requested=true 2024-12-08T01:14:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:06,678 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:14:06,680 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:06,680 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:06,680 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:06,680 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,681 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/d4c8f44ba1c24380ba4b09cee58d72ad, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/518663db9e144324b1738651337f481e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2164eaf1d4344f7cb2f07a801f084303] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.5 K 2024-12-08T01:14:06,681 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c8f44ba1c24380ba4b09cee58d72ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733620444324 2024-12-08T01:14:06,682 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 518663db9e144324b1738651337f481e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733620444941 2024-12-08T01:14:06,683 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:06,683 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2164eaf1d4344f7cb2f07a801f084303, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620445306 2024-12-08T01:14:06,683 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:06,683 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,683 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/91c131c7a3ac4417a1f7d15957f6b34a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0babee2f88cd4eabaf4653ab4340098e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5472b08cd7fa4b18be15c63d8fdc6c43] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=35.3 K 2024-12-08T01:14:06,686 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c131c7a3ac4417a1f7d15957f6b34a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733620444324 2024-12-08T01:14:06,686 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0babee2f88cd4eabaf4653ab4340098e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733620444941 2024-12-08T01:14:06,687 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5472b08cd7fa4b18be15c63d8fdc6c43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620445328 2024-12-08T01:14:06,704 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#18 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:06,706 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/f5fb77854ee04c71a1f8589542ba210a is 50, key is test_row_0/A:col10/1733620445356/Put/seqid=0 2024-12-08T01:14:06,715 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:06,716 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/cb90ad3890fa4392998069fc96a59c1a is 50, key is test_row_0/B:col10/1733620445356/Put/seqid=0 2024-12-08T01:14:06,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T01:14:06,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,720 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:14:06,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:06,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:06,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:06,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:06,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:06,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T01:14:06,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/4a8305da48154ad4b0356f366137a396 is 50, key is test_row_0/A:col10/1733620445729/Put/seqid=0 2024-12-08T01:14:06,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741855_1031 (size=12207) 2024-12-08T01:14:06,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741856_1032 (size=12207) 2024-12-08T01:14:06,763 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/f5fb77854ee04c71a1f8589542ba210a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/f5fb77854ee04c71a1f8589542ba210a 2024-12-08T01:14:06,778 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into f5fb77854ee04c71a1f8589542ba210a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:06,778 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:06,778 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620446678; duration=0sec 2024-12-08T01:14:06,780 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:06,780 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:06,780 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741857_1033 (size=12001) 2024-12-08T01:14:06,784 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/4a8305da48154ad4b0356f366137a396 2024-12-08T01:14:06,786 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:06,786 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:06,786 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:06,787 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6506dfaab68d4ade80895a9d07e9bdd0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3419f2b535334576b7f72e39ce5d5a58, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/f6e466e6f4a94225be8a0ed80bb74622] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=35.3 K 2024-12-08T01:14:06,791 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6506dfaab68d4ade80895a9d07e9bdd0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733620444324 2024-12-08T01:14:06,792 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3419f2b535334576b7f72e39ce5d5a58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733620444941 2024-12-08T01:14:06,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/93a809c19d6c4abfbee2e0b182136836 is 50, key is test_row_0/B:col10/1733620445729/Put/seqid=0 2024-12-08T01:14:06,801 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e466e6f4a94225be8a0ed80bb74622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620445328 2024-12-08T01:14:06,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741858_1034 (size=12001) 2024-12-08T01:14:06,821 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/93a809c19d6c4abfbee2e0b182136836 2024-12-08T01:14:06,834 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:06,836 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/fbccc9d429474b7f8a9613dde6aca519 is 50, key is test_row_0/C:col10/1733620445356/Put/seqid=0 2024-12-08T01:14:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3d2e6256b9ad400db4a75a02b5357eed is 50, key is test_row_0/C:col10/1733620445729/Put/seqid=0 2024-12-08T01:14:06,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741859_1035 (size=12207) 2024-12-08T01:14:06,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741860_1036 (size=12001) 2024-12-08T01:14:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:06,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:06,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620506934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620506939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620507042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620507052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,175 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/cb90ad3890fa4392998069fc96a59c1a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/cb90ad3890fa4392998069fc96a59c1a 2024-12-08T01:14:07,189 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into cb90ad3890fa4392998069fc96a59c1a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:07,189 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,189 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620446678; duration=0sec 2024-12-08T01:14:07,189 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:07,189 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:07,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620507253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620507262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,271 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/fbccc9d429474b7f8a9613dde6aca519 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fbccc9d429474b7f8a9613dde6aca519 2024-12-08T01:14:07,282 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3d2e6256b9ad400db4a75a02b5357eed 2024-12-08T01:14:07,284 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into fbccc9d429474b7f8a9613dde6aca519(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:07,284 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,284 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620446679; duration=0sec 2024-12-08T01:14:07,284 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:07,284 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/4a8305da48154ad4b0356f366137a396 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/4a8305da48154ad4b0356f366137a396 2024-12-08T01:14:07,304 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/4a8305da48154ad4b0356f366137a396, entries=150, sequenceid=103, filesize=11.7 K 2024-12-08T01:14:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/93a809c19d6c4abfbee2e0b182136836 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/93a809c19d6c4abfbee2e0b182136836 2024-12-08T01:14:07,322 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/93a809c19d6c4abfbee2e0b182136836, entries=150, sequenceid=103, filesize=11.7 K 2024-12-08T01:14:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3d2e6256b9ad400db4a75a02b5357eed as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3d2e6256b9ad400db4a75a02b5357eed 2024-12-08T01:14:07,336 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3d2e6256b9ad400db4a75a02b5357eed, entries=150, sequenceid=103, filesize=11.7 K 2024-12-08T01:14:07,338 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13ab931b3a97ca4195bbab8a55597446 in 619ms, sequenceid=103, compaction requested=false 2024-12-08T01:14:07,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:07,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-08T01:14:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-08T01:14:07,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-08T01:14:07,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7510 sec 2024-12-08T01:14:07,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7650 sec 2024-12-08T01:14:07,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:07,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/47a0e2f908c44de8bae74a0691b5b547 is 50, key is test_row_0/A:col10/1733620447563/Put/seqid=0 2024-12-08T01:14:07,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620507618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620507622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741861_1037 (size=14341) 2024-12-08T01:14:07,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/47a0e2f908c44de8bae74a0691b5b547 2024-12-08T01:14:07,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/05e6288a26fa4d3a901e357f7049d653 is 50, key is test_row_0/B:col10/1733620447563/Put/seqid=0 2024-12-08T01:14:07,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741862_1038 (size=12001) 2024-12-08T01:14:07,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/05e6288a26fa4d3a901e357f7049d653 2024-12-08T01:14:07,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/519c3cbf362744aab6fe2d0406370d91 is 50, key is test_row_0/C:col10/1733620447563/Put/seqid=0 2024-12-08T01:14:07,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741863_1039 (size=12001) 2024-12-08T01:14:07,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/519c3cbf362744aab6fe2d0406370d91 2024-12-08T01:14:07,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/47a0e2f908c44de8bae74a0691b5b547 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/47a0e2f908c44de8bae74a0691b5b547 2024-12-08T01:14:07,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620507732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620507732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T01:14:07,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:07,743 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-08T01:14:07,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T01:14:07,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:07,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:07,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/47a0e2f908c44de8bae74a0691b5b547, entries=200, sequenceid=121, filesize=14.0 K 2024-12-08T01:14:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/05e6288a26fa4d3a901e357f7049d653 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/05e6288a26fa4d3a901e357f7049d653 2024-12-08T01:14:07,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/05e6288a26fa4d3a901e357f7049d653, entries=150, sequenceid=121, filesize=11.7 K 2024-12-08T01:14:07,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/519c3cbf362744aab6fe2d0406370d91 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/519c3cbf362744aab6fe2d0406370d91 2024-12-08T01:14:07,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/519c3cbf362744aab6fe2d0406370d91, entries=150, sequenceid=121, filesize=11.7 K 2024-12-08T01:14:07,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 13ab931b3a97ca4195bbab8a55597446 in 228ms, sequenceid=121, compaction requested=true 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-08T01:14:07,797 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:07,797 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:07,800 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:07,800 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:07,800 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:07,800 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:07,800 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:07,800 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:07,801 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/f5fb77854ee04c71a1f8589542ba210a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/4a8305da48154ad4b0356f366137a396, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/47a0e2f908c44de8bae74a0691b5b547] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.6 K 2024-12-08T01:14:07,801 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fbccc9d429474b7f8a9613dde6aca519, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3d2e6256b9ad400db4a75a02b5357eed, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/519c3cbf362744aab6fe2d0406370d91] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=35.4 K 2024-12-08T01:14:07,802 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fbccc9d429474b7f8a9613dde6aca519, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620445328 2024-12-08T01:14:07,802 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5fb77854ee04c71a1f8589542ba210a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620445328 2024-12-08T01:14:07,803 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a8305da48154ad4b0356f366137a396, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733620445729 2024-12-08T01:14:07,803 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2e6256b9ad400db4a75a02b5357eed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733620445729 2024-12-08T01:14:07,804 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47a0e2f908c44de8bae74a0691b5b547, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733620446916 2024-12-08T01:14:07,804 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 519c3cbf362744aab6fe2d0406370d91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733620446916 2024-12-08T01:14:07,821 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:07,821 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/5a858d0f6bdf47eebc151e029aac04be is 50, key is test_row_0/C:col10/1733620447563/Put/seqid=0 2024-12-08T01:14:07,830 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:07,831 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/7dbd973bd15448049944e96cf7cef912 is 50, key is test_row_0/A:col10/1733620447563/Put/seqid=0 2024-12-08T01:14:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741864_1040 (size=12309) 2024-12-08T01:14:07,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741865_1041 (size=12309) 2024-12-08T01:14:07,861 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/5a858d0f6bdf47eebc151e029aac04be as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/5a858d0f6bdf47eebc151e029aac04be 2024-12-08T01:14:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T01:14:07,872 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/7dbd973bd15448049944e96cf7cef912 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/7dbd973bd15448049944e96cf7cef912 2024-12-08T01:14:07,881 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 5a858d0f6bdf47eebc151e029aac04be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:07,882 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,882 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620447795; duration=0sec 2024-12-08T01:14:07,882 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:07,882 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:07,882 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:07,884 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:07,884 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:07,884 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:07,885 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/cb90ad3890fa4392998069fc96a59c1a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/93a809c19d6c4abfbee2e0b182136836, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/05e6288a26fa4d3a901e357f7049d653] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=35.4 K 2024-12-08T01:14:07,885 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting cb90ad3890fa4392998069fc96a59c1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620445328 2024-12-08T01:14:07,886 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a809c19d6c4abfbee2e0b182136836, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733620445729 2024-12-08T01:14:07,887 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 05e6288a26fa4d3a901e357f7049d653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733620446916 2024-12-08T01:14:07,891 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into 7dbd973bd15448049944e96cf7cef912(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:07,891 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,891 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620447795; duration=0sec 2024-12-08T01:14:07,892 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:07,892 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:07,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T01:14:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:07,905 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:14:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:07,914 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:07,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a35eecb111b449259d2c2da1594d2f7c is 50, key is test_row_0/A:col10/1733620447611/Put/seqid=0 2024-12-08T01:14:07,914 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/dcb4df59c3c2442790b5af02ce000ceb is 50, key is test_row_0/B:col10/1733620447563/Put/seqid=0 2024-12-08T01:14:07,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741866_1042 (size=12151) 2024-12-08T01:14:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a35eecb111b449259d2c2da1594d2f7c 2024-12-08T01:14:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741867_1043 (size=12309) 2024-12-08T01:14:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:07,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:07,947 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/dcb4df59c3c2442790b5af02ce000ceb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/dcb4df59c3c2442790b5af02ce000ceb 2024-12-08T01:14:07,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5c2493ef71f440658ecbe895c1845f8b is 50, key is test_row_0/B:col10/1733620447611/Put/seqid=0 2024-12-08T01:14:07,962 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into dcb4df59c3c2442790b5af02ce000ceb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:07,963 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:07,963 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620447795; duration=0sec 2024-12-08T01:14:07,963 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:07,963 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620507988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620507988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741868_1044 (size=12151) 2024-12-08T01:14:08,002 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5c2493ef71f440658ecbe895c1845f8b 2024-12-08T01:14:08,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/42f6e0bba28241e983ef1a97d14b2d2e is 50, key is test_row_0/C:col10/1733620447611/Put/seqid=0 2024-12-08T01:14:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741869_1045 (size=12151) 2024-12-08T01:14:08,029 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/42f6e0bba28241e983ef1a97d14b2d2e 2024-12-08T01:14:08,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/a35eecb111b449259d2c2da1594d2f7c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a35eecb111b449259d2c2da1594d2f7c 2024-12-08T01:14:08,054 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a35eecb111b449259d2c2da1594d2f7c, entries=150, sequenceid=140, filesize=11.9 K 2024-12-08T01:14:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5c2493ef71f440658ecbe895c1845f8b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c2493ef71f440658ecbe895c1845f8b 2024-12-08T01:14:08,071 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c2493ef71f440658ecbe895c1845f8b, entries=150, sequenceid=140, filesize=11.9 K 2024-12-08T01:14:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T01:14:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/42f6e0bba28241e983ef1a97d14b2d2e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/42f6e0bba28241e983ef1a97d14b2d2e 2024-12-08T01:14:08,082 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/42f6e0bba28241e983ef1a97d14b2d2e, entries=150, sequenceid=140, filesize=11.9 K 2024-12-08T01:14:08,087 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 13ab931b3a97ca4195bbab8a55597446 in 182ms, sequenceid=140, compaction requested=false 2024-12-08T01:14:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-08T01:14:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-08T01:14:08,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T01:14:08,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 343 msec 2024-12-08T01:14:08,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 357 msec 2024-12-08T01:14:08,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:14:08,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:08,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:08,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:08,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:08,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:08,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:08,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/08a178f1f7e541a3bda751bc883507bd is 50, key is test_row_0/A:col10/1733620447964/Put/seqid=0 2024-12-08T01:14:08,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741870_1046 (size=16931) 2024-12-08T01:14:08,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/08a178f1f7e541a3bda751bc883507bd 2024-12-08T01:14:08,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620508148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620508155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/2e643b294d634b42a0cbadcd838587b5 is 50, key is test_row_0/B:col10/1733620447964/Put/seqid=0 2024-12-08T01:14:08,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741871_1047 (size=12151) 2024-12-08T01:14:08,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/2e643b294d634b42a0cbadcd838587b5 2024-12-08T01:14:08,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/0d54a722743f48dc831778120ef5358f is 50, key is test_row_0/C:col10/1733620447964/Put/seqid=0 2024-12-08T01:14:08,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741872_1048 (size=12151) 2024-12-08T01:14:08,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/0d54a722743f48dc831778120ef5358f 2024-12-08T01:14:08,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/08a178f1f7e541a3bda751bc883507bd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/08a178f1f7e541a3bda751bc883507bd 2024-12-08T01:14:08,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/08a178f1f7e541a3bda751bc883507bd, entries=250, sequenceid=159, filesize=16.5 K 2024-12-08T01:14:08,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620508262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620508263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/2e643b294d634b42a0cbadcd838587b5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/2e643b294d634b42a0cbadcd838587b5 2024-12-08T01:14:08,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/2e643b294d634b42a0cbadcd838587b5, entries=150, sequenceid=159, filesize=11.9 K 2024-12-08T01:14:08,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/0d54a722743f48dc831778120ef5358f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0d54a722743f48dc831778120ef5358f 2024-12-08T01:14:08,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0d54a722743f48dc831778120ef5358f, entries=150, sequenceid=159, filesize=11.9 K 2024-12-08T01:14:08,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13ab931b3a97ca4195bbab8a55597446 in 178ms, sequenceid=159, compaction requested=true 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-08T01:14:08,289 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:08,290 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:08,291 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:08,291 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:08,291 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,291 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/7dbd973bd15448049944e96cf7cef912, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a35eecb111b449259d2c2da1594d2f7c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/08a178f1f7e541a3bda751bc883507bd] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=40.4 K 2024-12-08T01:14:08,291 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:08,291 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:08,292 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,292 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/5a858d0f6bdf47eebc151e029aac04be, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/42f6e0bba28241e983ef1a97d14b2d2e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0d54a722743f48dc831778120ef5358f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=35.8 K 2024-12-08T01:14:08,292 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dbd973bd15448049944e96cf7cef912, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733620446916 2024-12-08T01:14:08,292 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a858d0f6bdf47eebc151e029aac04be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733620446916 2024-12-08T01:14:08,292 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35eecb111b449259d2c2da1594d2f7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733620447609 2024-12-08T01:14:08,293 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08a178f1f7e541a3bda751bc883507bd, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733620447964 2024-12-08T01:14:08,293 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f6e0bba28241e983ef1a97d14b2d2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733620447609 2024-12-08T01:14:08,294 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d54a722743f48dc831778120ef5358f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733620447964 2024-12-08T01:14:08,313 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:08,314 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/785469833ad245bfa84557d18351e4f1 is 50, key is test_row_0/A:col10/1733620447964/Put/seqid=0 2024-12-08T01:14:08,316 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:08,317 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/e9201be6105f46fe909fc41b7ddebeee is 50, key is test_row_0/C:col10/1733620447964/Put/seqid=0 2024-12-08T01:14:08,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741874_1050 (size=12561) 2024-12-08T01:14:08,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741873_1049 (size=12561) 2024-12-08T01:14:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T01:14:08,383 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:08,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-08T01:14:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T01:14:08,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:08,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:08,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:08,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:08,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:08,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/1fed5dfc0d9e4474be3ae34b949a159b is 50, key is test_row_0/A:col10/1733620448145/Put/seqid=0 2024-12-08T01:14:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T01:14:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741875_1051 (size=16931) 2024-12-08T01:14:08,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/1fed5dfc0d9e4474be3ae34b949a159b 2024-12-08T01:14:08,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/4aacf05b158d4cc6aace4059f1adde6e is 50, key is test_row_0/B:col10/1733620448145/Put/seqid=0 2024-12-08T01:14:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620508522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620508523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,531 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620508530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620508532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,536 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:08,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741876_1052 (size=12151) 2024-12-08T01:14:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620508542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,547 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620508642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620508643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T01:14:08,772 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/e9201be6105f46fe909fc41b7ddebeee as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/e9201be6105f46fe909fc41b7ddebeee 2024-12-08T01:14:08,773 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/785469833ad245bfa84557d18351e4f1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/785469833ad245bfa84557d18351e4f1 2024-12-08T01:14:08,786 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into e9201be6105f46fe909fc41b7ddebeee(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:08,786 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:08,786 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620448289; duration=0sec 2024-12-08T01:14:08,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:08,787 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into 785469833ad245bfa84557d18351e4f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:08,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:08,787 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:08,787 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620448289; duration=0sec 2024-12-08T01:14:08,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:08,787 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:08,787 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:08,791 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:08,791 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:08,791 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,792 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/dcb4df59c3c2442790b5af02ce000ceb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c2493ef71f440658ecbe895c1845f8b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/2e643b294d634b42a0cbadcd838587b5] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=35.8 K 2024-12-08T01:14:08,792 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb4df59c3c2442790b5af02ce000ceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733620446916 2024-12-08T01:14:08,793 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2493ef71f440658ecbe895c1845f8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733620447609 2024-12-08T01:14:08,793 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e643b294d634b42a0cbadcd838587b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733620447964 2024-12-08T01:14:08,809 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:08,810 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/692f7d19f12d43bf99f2dd747f3cedf8 is 50, key is test_row_0/B:col10/1733620447964/Put/seqid=0 2024-12-08T01:14:08,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741877_1053 (size=12561) 2024-12-08T01:14:08,830 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/692f7d19f12d43bf99f2dd747f3cedf8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/692f7d19f12d43bf99f2dd747f3cedf8 2024-12-08T01:14:08,838 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 692f7d19f12d43bf99f2dd747f3cedf8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:08,838 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:08,838 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620448289; duration=0sec 2024-12-08T01:14:08,838 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:08,838 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:08,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:08,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620508852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:08,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620508852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:08,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/4aacf05b158d4cc6aace4059f1adde6e 2024-12-08T01:14:08,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/90b435b810ba40bd81461e067c947a30 is 50, key is test_row_0/C:col10/1733620448145/Put/seqid=0 2024-12-08T01:14:08,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741878_1054 (size=12151) 2024-12-08T01:14:09,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:09,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:09,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T01:14:09,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:09,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:09,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620509162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620509163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:09,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:09,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/90b435b810ba40bd81461e067c947a30 2024-12-08T01:14:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/1fed5dfc0d9e4474be3ae34b949a159b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/1fed5dfc0d9e4474be3ae34b949a159b 2024-12-08T01:14:09,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/1fed5dfc0d9e4474be3ae34b949a159b, entries=250, sequenceid=175, filesize=16.5 K 2024-12-08T01:14:09,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/4aacf05b158d4cc6aace4059f1adde6e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4aacf05b158d4cc6aace4059f1adde6e 2024-12-08T01:14:09,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4aacf05b158d4cc6aace4059f1adde6e, entries=150, sequenceid=175, filesize=11.9 K 2024-12-08T01:14:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/90b435b810ba40bd81461e067c947a30 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/90b435b810ba40bd81461e067c947a30 2024-12-08T01:14:09,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/90b435b810ba40bd81461e067c947a30, entries=150, sequenceid=175, filesize=11.9 K 2024-12-08T01:14:09,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 13ab931b3a97ca4195bbab8a55597446 in 943ms, sequenceid=175, compaction requested=false 2024-12-08T01:14:09,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:09,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T01:14:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,471 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c94f364efc654614ba2e2d3cd07edc8b is 50, key is test_row_0/A:col10/1733620448520/Put/seqid=0 2024-12-08T01:14:09,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741879_1055 (size=12151) 2024-12-08T01:14:09,494 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c94f364efc654614ba2e2d3cd07edc8b 2024-12-08T01:14:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/350d281d50c34dcea88f3b08afcf6e30 is 50, key is test_row_0/B:col10/1733620448520/Put/seqid=0 2024-12-08T01:14:09,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741880_1056 (size=12151) 2024-12-08T01:14:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T01:14:09,523 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/350d281d50c34dcea88f3b08afcf6e30 2024-12-08T01:14:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/c0129325fac443349644adef48abae71 is 50, key is test_row_0/C:col10/1733620448520/Put/seqid=0 2024-12-08T01:14:09,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741881_1057 (size=12151) 2024-12-08T01:14:09,545 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/c0129325fac443349644adef48abae71 2024-12-08T01:14:09,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c94f364efc654614ba2e2d3cd07edc8b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c94f364efc654614ba2e2d3cd07edc8b 2024-12-08T01:14:09,566 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c94f364efc654614ba2e2d3cd07edc8b, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T01:14:09,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/350d281d50c34dcea88f3b08afcf6e30 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/350d281d50c34dcea88f3b08afcf6e30 2024-12-08T01:14:09,578 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/350d281d50c34dcea88f3b08afcf6e30, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T01:14:09,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/c0129325fac443349644adef48abae71 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/c0129325fac443349644adef48abae71 2024-12-08T01:14:09,589 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/c0129325fac443349644adef48abae71, entries=150, sequenceid=194, filesize=11.9 K 2024-12-08T01:14:09,592 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 13ab931b3a97ca4195bbab8a55597446 in 120ms, sequenceid=194, compaction requested=true 2024-12-08T01:14:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-08T01:14:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-08T01:14:09,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T01:14:09,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2050 sec 2024-12-08T01:14:09,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2180 sec 2024-12-08T01:14:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:09,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:09,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/3729978528244dd493610ba8a08a03f9 is 50, key is test_row_0/A:col10/1733620449683/Put/seqid=0 2024-12-08T01:14:09,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741882_1058 (size=14541) 2024-12-08T01:14:09,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/3729978528244dd493610ba8a08a03f9 2024-12-08T01:14:09,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/c1fac3f7f8874850ba8a8e9883e01e95 is 50, key is test_row_0/B:col10/1733620449683/Put/seqid=0 2024-12-08T01:14:09,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741883_1059 (size=12151) 2024-12-08T01:14:09,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620449741, max timestamp=1733620449773Num uniques 859; 2024-12-08T01:14:09,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:09,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620449741, max timestamp=1733620449773Num uniques 859; 2024-12-08T01:14:09,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:09,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620449741, max timestamp=1733620449773Num uniques 859; 2024-12-08T01:14:09,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:09,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620509864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620509873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:09,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:09,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620509982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:09,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620509992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:10,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/c1fac3f7f8874850ba8a8e9883e01e95 2024-12-08T01:14:10,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/be51d3f57ed74f30b1e04ae5b17eac07 is 50, key is test_row_0/C:col10/1733620449683/Put/seqid=0 2024-12-08T01:14:10,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741884_1060 (size=12151) 2024-12-08T01:14:10,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620510192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620510212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:10,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620510502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:10,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620510521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T01:14:10,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:10,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-08T01:14:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T01:14:10,545 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:10,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:10,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:10,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/be51d3f57ed74f30b1e04ae5b17eac07 2024-12-08T01:14:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/3729978528244dd493610ba8a08a03f9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/3729978528244dd493610ba8a08a03f9 2024-12-08T01:14:10,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/3729978528244dd493610ba8a08a03f9, entries=200, sequenceid=204, filesize=14.2 K 2024-12-08T01:14:10,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/c1fac3f7f8874850ba8a8e9883e01e95 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/c1fac3f7f8874850ba8a8e9883e01e95 2024-12-08T01:14:10,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/c1fac3f7f8874850ba8a8e9883e01e95, entries=150, sequenceid=204, filesize=11.9 K 2024-12-08T01:14:10,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/be51d3f57ed74f30b1e04ae5b17eac07 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/be51d3f57ed74f30b1e04ae5b17eac07 2024-12-08T01:14:10,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/be51d3f57ed74f30b1e04ae5b17eac07, entries=150, sequenceid=204, filesize=11.9 K 2024-12-08T01:14:10,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13ab931b3a97ca4195bbab8a55597446 in 921ms, sequenceid=204, compaction requested=true 2024-12-08T01:14:10,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:10,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:10,613 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:10,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:10,613 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:10,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:10,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:10,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:10,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:10,616 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:10,616 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56184 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:10,616 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:10,616 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:10,617 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:10,617 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:10,617 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/785469833ad245bfa84557d18351e4f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/1fed5dfc0d9e4474be3ae34b949a159b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c94f364efc654614ba2e2d3cd07edc8b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/3729978528244dd493610ba8a08a03f9] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=54.9 K 2024-12-08T01:14:10,617 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/692f7d19f12d43bf99f2dd747f3cedf8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4aacf05b158d4cc6aace4059f1adde6e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/350d281d50c34dcea88f3b08afcf6e30, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/c1fac3f7f8874850ba8a8e9883e01e95] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=47.9 K 2024-12-08T01:14:10,617 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 692f7d19f12d43bf99f2dd747f3cedf8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733620447964 2024-12-08T01:14:10,617 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 785469833ad245bfa84557d18351e4f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733620447964 2024-12-08T01:14:10,618 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aacf05b158d4cc6aace4059f1adde6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733620448139 2024-12-08T01:14:10,618 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fed5dfc0d9e4474be3ae34b949a159b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733620448139 2024-12-08T01:14:10,618 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 350d281d50c34dcea88f3b08afcf6e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733620448517 2024-12-08T01:14:10,618 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94f364efc654614ba2e2d3cd07edc8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733620448517 2024-12-08T01:14:10,618 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3729978528244dd493610ba8a08a03f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733620449682 2024-12-08T01:14:10,619 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c1fac3f7f8874850ba8a8e9883e01e95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733620449682 2024-12-08T01:14:10,630 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:10,631 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/8f143455530d40229fc04cf193235aa5 is 50, key is test_row_0/B:col10/1733620449683/Put/seqid=0 2024-12-08T01:14:10,634 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:10,635 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/de86b5bb9f8e46e29ce7a821165e4c1e is 50, key is test_row_0/A:col10/1733620449683/Put/seqid=0 2024-12-08T01:14:10,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741886_1062 (size=12697) 2024-12-08T01:14:10,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741885_1061 (size=12697) 2024-12-08T01:14:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T01:14:10,659 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/de86b5bb9f8e46e29ce7a821165e4c1e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/de86b5bb9f8e46e29ce7a821165e4c1e 2024-12-08T01:14:10,668 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into de86b5bb9f8e46e29ce7a821165e4c1e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:10,669 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:10,669 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=12, startTime=1733620450613; duration=0sec 2024-12-08T01:14:10,669 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:10,669 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:10,669 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:10,671 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:10,671 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:10,672 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:10,672 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/e9201be6105f46fe909fc41b7ddebeee, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/90b435b810ba40bd81461e067c947a30, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/c0129325fac443349644adef48abae71, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/be51d3f57ed74f30b1e04ae5b17eac07] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=47.9 K 2024-12-08T01:14:10,672 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9201be6105f46fe909fc41b7ddebeee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733620447964 2024-12-08T01:14:10,673 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90b435b810ba40bd81461e067c947a30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733620448139 2024-12-08T01:14:10,673 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0129325fac443349644adef48abae71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733620448517 2024-12-08T01:14:10,674 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting be51d3f57ed74f30b1e04ae5b17eac07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733620449682 2024-12-08T01:14:10,684 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:10,685 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/732e2390bb72452bb05702fd367d7421 is 50, key is test_row_0/C:col10/1733620449683/Put/seqid=0 2024-12-08T01:14:10,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741887_1063 (size=12697) 2024-12-08T01:14:10,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T01:14:10,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:10,701 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T01:14:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:10,713 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/732e2390bb72452bb05702fd367d7421 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/732e2390bb72452bb05702fd367d7421 2024-12-08T01:14:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/809974d8936643fc9e59af0bd2108890 is 50, key is test_row_0/A:col10/1733620449844/Put/seqid=0 2024-12-08T01:14:10,722 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 732e2390bb72452bb05702fd367d7421(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:10,722 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:10,722 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=12, startTime=1733620450613; duration=0sec 2024-12-08T01:14:10,723 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:10,723 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:10,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741888_1064 (size=12151) 2024-12-08T01:14:10,729 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/809974d8936643fc9e59af0bd2108890 2024-12-08T01:14:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5dea50cbee594b368fb4f6cab6ae4b6b is 50, key is test_row_0/B:col10/1733620449844/Put/seqid=0 2024-12-08T01:14:10,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741889_1065 (size=12151) 2024-12-08T01:14:10,767 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5dea50cbee594b368fb4f6cab6ae4b6b 2024-12-08T01:14:10,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b34b32f0e34545dd961f51d0d0b08978 is 50, key is test_row_0/C:col10/1733620449844/Put/seqid=0 2024-12-08T01:14:10,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741890_1066 (size=12151) 2024-12-08T01:14:10,802 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b34b32f0e34545dd961f51d0d0b08978 2024-12-08T01:14:10,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/809974d8936643fc9e59af0bd2108890 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/809974d8936643fc9e59af0bd2108890 2024-12-08T01:14:10,823 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/809974d8936643fc9e59af0bd2108890, entries=150, sequenceid=242, filesize=11.9 K 2024-12-08T01:14:10,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5dea50cbee594b368fb4f6cab6ae4b6b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5dea50cbee594b368fb4f6cab6ae4b6b 2024-12-08T01:14:10,836 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5dea50cbee594b368fb4f6cab6ae4b6b, entries=150, sequenceid=242, filesize=11.9 K 2024-12-08T01:14:10,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b34b32f0e34545dd961f51d0d0b08978 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b34b32f0e34545dd961f51d0d0b08978 2024-12-08T01:14:10,846 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b34b32f0e34545dd961f51d0d0b08978, entries=150, sequenceid=242, filesize=11.9 K 2024-12-08T01:14:10,851 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 13ab931b3a97ca4195bbab8a55597446 in 149ms, sequenceid=242, compaction requested=false 2024-12-08T01:14:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-08T01:14:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-08T01:14:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T01:14:10,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-08T01:14:10,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 310 msec 2024-12-08T01:14:10,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 333 msec 2024-12-08T01:14:11,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:11,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:11,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c8f1cd862f4f49c6b3ea5c1d612cf635 is 50, key is test_row_1/A:col10/1733620451021/Put/seqid=0 2024-12-08T01:14:11,078 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/8f143455530d40229fc04cf193235aa5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/8f143455530d40229fc04cf193235aa5 2024-12-08T01:14:11,091 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 8f143455530d40229fc04cf193235aa5(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:11,091 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:11,091 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=12, startTime=1733620450613; duration=0sec 2024-12-08T01:14:11,091 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:11,091 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741891_1067 (size=9757) 2024-12-08T01:14:11,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620451078, max timestamp=1733620451098Num uniques 859; 2024-12-08T01:14:11,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:11,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c8f1cd862f4f49c6b3ea5c1d612cf635 2024-12-08T01:14:11,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620451078, max timestamp=1733620451098Num uniques 859; 2024-12-08T01:14:11,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:11,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620451078, max timestamp=1733620451107Num uniques 859; 2024-12-08T01:14:11,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:11,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7450150cd68d41f7ac8f3589a92aa8f8 is 50, key is test_row_1/B:col10/1733620451021/Put/seqid=0 2024-12-08T01:14:11,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741892_1068 (size=9757) 2024-12-08T01:14:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T01:14:11,172 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:11,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-08T01:14:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T01:14:11,176 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:11,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:11,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620511172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620511175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T01:14:11,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620511292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620511292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T01:14:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:11,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T01:14:11,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T01:14:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620511502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620511502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7450150cd68d41f7ac8f3589a92aa8f8 2024-12-08T01:14:11,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/8804c50b361b42cb958e46135d01f4ef is 50, key is test_row_1/C:col10/1733620451021/Put/seqid=0 2024-12-08T01:14:11,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741893_1069 (size=9757) 2024-12-08T01:14:11,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/8804c50b361b42cb958e46135d01f4ef 2024-12-08T01:14:11,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c8f1cd862f4f49c6b3ea5c1d612cf635 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c8f1cd862f4f49c6b3ea5c1d612cf635 2024-12-08T01:14:11,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c8f1cd862f4f49c6b3ea5c1d612cf635, entries=100, sequenceid=253, filesize=9.5 K 2024-12-08T01:14:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7450150cd68d41f7ac8f3589a92aa8f8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7450150cd68d41f7ac8f3589a92aa8f8 2024-12-08T01:14:11,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7450150cd68d41f7ac8f3589a92aa8f8, entries=100, sequenceid=253, filesize=9.5 K 2024-12-08T01:14:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/8804c50b361b42cb958e46135d01f4ef as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/8804c50b361b42cb958e46135d01f4ef 2024-12-08T01:14:11,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/8804c50b361b42cb958e46135d01f4ef, entries=100, sequenceid=253, filesize=9.5 K 2024-12-08T01:14:11,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 13ab931b3a97ca4195bbab8a55597446 in 569ms, sequenceid=253, compaction requested=true 2024-12-08T01:14:11,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:11,604 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:11,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:11,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:11,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:11,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:11,604 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:11,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:11,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:11,605 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:11,605 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:11,605 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:11,605 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,605 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:11,605 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/de86b5bb9f8e46e29ce7a821165e4c1e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/809974d8936643fc9e59af0bd2108890, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c8f1cd862f4f49c6b3ea5c1d612cf635] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=33.8 K 2024-12-08T01:14:11,605 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,605 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/8f143455530d40229fc04cf193235aa5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5dea50cbee594b368fb4f6cab6ae4b6b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7450150cd68d41f7ac8f3589a92aa8f8] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=33.8 K 2024-12-08T01:14:11,605 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting de86b5bb9f8e46e29ce7a821165e4c1e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733620449682 2024-12-08T01:14:11,606 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f143455530d40229fc04cf193235aa5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733620449682 2024-12-08T01:14:11,606 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 809974d8936643fc9e59af0bd2108890, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733620449844 2024-12-08T01:14:11,606 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dea50cbee594b368fb4f6cab6ae4b6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733620449844 2024-12-08T01:14:11,606 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8f1cd862f4f49c6b3ea5c1d612cf635, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620451021 2024-12-08T01:14:11,606 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7450150cd68d41f7ac8f3589a92aa8f8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620451021 2024-12-08T01:14:11,615 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:11,616 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5c8c101a5d5c422286810775582fccf0 is 50, key is test_row_0/B:col10/1733620449844/Put/seqid=0 2024-12-08T01:14:11,618 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:11,618 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/791fff8037254445b52a149e287af4fd is 50, key is test_row_0/A:col10/1733620449844/Put/seqid=0 2024-12-08T01:14:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741894_1070 (size=12799) 2024-12-08T01:14:11,635 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/5c8c101a5d5c422286810775582fccf0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c8c101a5d5c422286810775582fccf0 2024-12-08T01:14:11,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T01:14:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,641 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-08T01:14:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741895_1071 (size=12799) 2024-12-08T01:14:11,651 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 5c8c101a5d5c422286810775582fccf0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:11,651 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:11,651 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620451604; duration=0sec 2024-12-08T01:14:11,652 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:11,652 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:11,652 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6074539b8798424792973e489985a566 is 50, key is test_row_0/A:col10/1733620451133/Put/seqid=0 2024-12-08T01:14:11,656 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:11,656 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:11,656 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,656 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/732e2390bb72452bb05702fd367d7421, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b34b32f0e34545dd961f51d0d0b08978, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/8804c50b361b42cb958e46135d01f4ef] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=33.8 K 2024-12-08T01:14:11,657 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 732e2390bb72452bb05702fd367d7421, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733620449682 2024-12-08T01:14:11,657 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b34b32f0e34545dd961f51d0d0b08978, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733620449844 2024-12-08T01:14:11,658 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8804c50b361b42cb958e46135d01f4ef, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620451021 2024-12-08T01:14:11,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741896_1072 (size=12301) 2024-12-08T01:14:11,669 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6074539b8798424792973e489985a566 2024-12-08T01:14:11,687 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:11,688 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3b81e83a280e4d2e9278098cac928410 is 50, key is test_row_0/C:col10/1733620449844/Put/seqid=0 2024-12-08T01:14:11,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/fa3a0c75ef014001ab17706f0481ead1 is 50, key is test_row_0/B:col10/1733620451133/Put/seqid=0 2024-12-08T01:14:11,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741897_1073 (size=12799) 2024-12-08T01:14:11,720 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/3b81e83a280e4d2e9278098cac928410 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3b81e83a280e4d2e9278098cac928410 2024-12-08T01:14:11,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741898_1074 (size=12301) 2024-12-08T01:14:11,732 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/fa3a0c75ef014001ab17706f0481ead1 2024-12-08T01:14:11,734 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 3b81e83a280e4d2e9278098cac928410(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:11,734 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:11,734 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620451604; duration=0sec 2024-12-08T01:14:11,735 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:11,735 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:11,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/1909c711be724f3ab985e9a4caebb93e is 50, key is test_row_0/C:col10/1733620451133/Put/seqid=0 2024-12-08T01:14:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741899_1075 (size=12301) 2024-12-08T01:14:11,775 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/1909c711be724f3ab985e9a4caebb93e 2024-12-08T01:14:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6074539b8798424792973e489985a566 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6074539b8798424792973e489985a566 2024-12-08T01:14:11,791 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6074539b8798424792973e489985a566, entries=150, sequenceid=291, filesize=12.0 K 2024-12-08T01:14:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/fa3a0c75ef014001ab17706f0481ead1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/fa3a0c75ef014001ab17706f0481ead1 2024-12-08T01:14:11,801 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/fa3a0c75ef014001ab17706f0481ead1, entries=150, sequenceid=291, filesize=12.0 K 2024-12-08T01:14:11,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/1909c711be724f3ab985e9a4caebb93e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/1909c711be724f3ab985e9a4caebb93e 2024-12-08T01:14:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T01:14:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:11,815 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/1909c711be724f3ab985e9a4caebb93e, entries=150, sequenceid=291, filesize=12.0 K 2024-12-08T01:14:11,818 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=13.42 KB/13740 for 13ab931b3a97ca4195bbab8a55597446 in 177ms, sequenceid=291, compaction requested=false 2024-12-08T01:14:11,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:11,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:11,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-08T01:14:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-08T01:14:11,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T01:14:11,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-12-08T01:14:11,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 655 msec 2024-12-08T01:14:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:11,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:11,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:11,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:11,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/8bbce9e860bb4a1da16be44a47185da2 is 50, key is test_row_0/A:col10/1733620451831/Put/seqid=0 2024-12-08T01:14:11,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741900_1076 (size=14741) 2024-12-08T01:14:11,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/8bbce9e860bb4a1da16be44a47185da2 2024-12-08T01:14:11,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/1413c017144a4f0698be311556b6c5b3 is 50, key is test_row_0/B:col10/1733620451831/Put/seqid=0 2024-12-08T01:14:11,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620451879, max timestamp=1733620451897Num uniques 859; 2024-12-08T01:14:11,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:11,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620451879, max timestamp=1733620451897Num uniques 859; 2024-12-08T01:14:11,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:11,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620451879, max timestamp=1733620451897Num uniques 859; 2024-12-08T01:14:11,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:11,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741901_1077 (size=12301) 2024-12-08T01:14:11,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/1413c017144a4f0698be311556b6c5b3 2024-12-08T01:14:11,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/4c96dd09e60f4845bad37ec253e7241f is 50, key is test_row_0/C:col10/1733620451831/Put/seqid=0 2024-12-08T01:14:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741902_1078 (size=12301) 2024-12-08T01:14:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620511977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:11,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620511983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,054 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/791fff8037254445b52a149e287af4fd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/791fff8037254445b52a149e287af4fd 2024-12-08T01:14:12,063 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into 791fff8037254445b52a149e287af4fd(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:12,063 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:12,064 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620451603; duration=0sec 2024-12-08T01:14:12,064 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:12,064 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:12,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620512092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620512101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620512302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T01:14:12,312 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:12,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-08T01:14:12,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620512311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T01:14:12,314 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:12,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:12,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:12,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/4c96dd09e60f4845bad37ec253e7241f 2024-12-08T01:14:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/8bbce9e860bb4a1da16be44a47185da2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8bbce9e860bb4a1da16be44a47185da2 2024-12-08T01:14:12,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8bbce9e860bb4a1da16be44a47185da2, entries=200, sequenceid=303, filesize=14.4 K 2024-12-08T01:14:12,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/1413c017144a4f0698be311556b6c5b3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/1413c017144a4f0698be311556b6c5b3 2024-12-08T01:14:12,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/1413c017144a4f0698be311556b6c5b3, entries=150, sequenceid=303, filesize=12.0 K 2024-12-08T01:14:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/4c96dd09e60f4845bad37ec253e7241f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/4c96dd09e60f4845bad37ec253e7241f 2024-12-08T01:14:12,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/4c96dd09e60f4845bad37ec253e7241f, entries=150, sequenceid=303, filesize=12.0 K 2024-12-08T01:14:12,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13ab931b3a97ca4195bbab8a55597446 in 575ms, sequenceid=303, compaction requested=true 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:12,414 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:12,414 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:12,415 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:12,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:12,416 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:12,416 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c8c101a5d5c422286810775582fccf0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/fa3a0c75ef014001ab17706f0481ead1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/1413c017144a4f0698be311556b6c5b3] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=36.5 K 2024-12-08T01:14:12,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c8c101a5d5c422286810775582fccf0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620449844 2024-12-08T01:14:12,416 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:12,417 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:12,417 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:12,417 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/791fff8037254445b52a149e287af4fd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6074539b8798424792973e489985a566, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8bbce9e860bb4a1da16be44a47185da2] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=38.9 K 2024-12-08T01:14:12,417 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3a0c75ef014001ab17706f0481ead1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733620451133 2024-12-08T01:14:12,417 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 791fff8037254445b52a149e287af4fd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620449844 2024-12-08T01:14:12,418 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1413c017144a4f0698be311556b6c5b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733620451831 2024-12-08T01:14:12,418 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6074539b8798424792973e489985a566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733620451133 2024-12-08T01:14:12,419 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbce9e860bb4a1da16be44a47185da2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733620451823 2024-12-08T01:14:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T01:14:12,433 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:12,433 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:12,433 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c304cb59ac31464ca47d5ae6a8da18b0 is 50, key is test_row_0/A:col10/1733620451831/Put/seqid=0 2024-12-08T01:14:12,434 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/585c8b9a942948c38d1ac3bcda71549e is 50, key is test_row_0/B:col10/1733620451831/Put/seqid=0 2024-12-08T01:14:12,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741904_1080 (size=13051) 2024-12-08T01:14:12,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T01:14:12,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:12,469 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T01:14:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:12,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741903_1079 (size=13051) 2024-12-08T01:14:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/0c33f9e08ba742c496c3b24b54d9f757 is 50, key is test_row_0/A:col10/1733620451949/Put/seqid=0 2024-12-08T01:14:12,481 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/585c8b9a942948c38d1ac3bcda71549e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/585c8b9a942948c38d1ac3bcda71549e 2024-12-08T01:14:12,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741905_1081 (size=12301) 2024-12-08T01:14:12,489 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 585c8b9a942948c38d1ac3bcda71549e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:12,489 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:12,489 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620452414; duration=0sec 2024-12-08T01:14:12,489 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:12,489 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:12,490 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:12,491 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:12,491 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:12,491 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:12,491 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3b81e83a280e4d2e9278098cac928410, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/1909c711be724f3ab985e9a4caebb93e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/4c96dd09e60f4845bad37ec253e7241f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=36.5 K 2024-12-08T01:14:12,492 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b81e83a280e4d2e9278098cac928410, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620449844 2024-12-08T01:14:12,492 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1909c711be724f3ab985e9a4caebb93e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733620451133 2024-12-08T01:14:12,492 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c96dd09e60f4845bad37ec253e7241f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733620451831 2024-12-08T01:14:12,505 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:12,506 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b97ef089ab6a45d1853384ba5d0bfda5 is 50, key is test_row_0/C:col10/1733620451831/Put/seqid=0 2024-12-08T01:14:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741906_1082 (size=13051) 2024-12-08T01:14:12,526 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b97ef089ab6a45d1853384ba5d0bfda5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b97ef089ab6a45d1853384ba5d0bfda5 2024-12-08T01:14:12,542 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into b97ef089ab6a45d1853384ba5d0bfda5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:12,542 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:12,542 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620452414; duration=0sec 2024-12-08T01:14:12,543 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:12,543 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:12,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620512576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620512580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620512582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,585 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620512612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620512622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T01:14:12,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620512691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620512692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,860 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/c304cb59ac31464ca47d5ae6a8da18b0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c304cb59ac31464ca47d5ae6a8da18b0 2024-12-08T01:14:12,866 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into c304cb59ac31464ca47d5ae6a8da18b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:12,866 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:12,867 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620452414; duration=0sec 2024-12-08T01:14:12,867 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:12,867 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:12,884 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/0c33f9e08ba742c496c3b24b54d9f757 2024-12-08T01:14:12,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7d0795422ce346d38bf0b0809f9e55f9 is 50, key is test_row_0/B:col10/1733620451949/Put/seqid=0 2024-12-08T01:14:12,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741907_1083 (size=12301) 2024-12-08T01:14:12,903 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7d0795422ce346d38bf0b0809f9e55f9 2024-12-08T01:14:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620512902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620512902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:12,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/fa6ad2f57a744011bac9f8543241cb77 is 50, key is test_row_0/C:col10/1733620451949/Put/seqid=0 2024-12-08T01:14:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741908_1084 (size=12301) 2024-12-08T01:14:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T01:14:13,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620513122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620513132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620513212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620513212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,325 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/fa6ad2f57a744011bac9f8543241cb77 2024-12-08T01:14:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/0c33f9e08ba742c496c3b24b54d9f757 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0c33f9e08ba742c496c3b24b54d9f757 2024-12-08T01:14:13,336 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0c33f9e08ba742c496c3b24b54d9f757, entries=150, sequenceid=341, filesize=12.0 K 2024-12-08T01:14:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7d0795422ce346d38bf0b0809f9e55f9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7d0795422ce346d38bf0b0809f9e55f9 2024-12-08T01:14:13,343 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7d0795422ce346d38bf0b0809f9e55f9, entries=150, sequenceid=341, filesize=12.0 K 2024-12-08T01:14:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/fa6ad2f57a744011bac9f8543241cb77 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fa6ad2f57a744011bac9f8543241cb77 2024-12-08T01:14:13,350 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fa6ad2f57a744011bac9f8543241cb77, entries=150, sequenceid=341, filesize=12.0 K 2024-12-08T01:14:13,351 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 13ab931b3a97ca4195bbab8a55597446 in 883ms, sequenceid=341, compaction requested=false 2024-12-08T01:14:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:13,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-08T01:14:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-08T01:14:13,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T01:14:13,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0380 sec 2024-12-08T01:14:13,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0450 sec 2024-12-08T01:14:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T01:14:13,452 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:13,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-08T01:14:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T01:14:13,454 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:13,455 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:13,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T01:14:13,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T01:14:13,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:13,608 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:13,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/fadcaef66b8346c5a096cd972cb31de2 is 50, key is test_row_0/A:col10/1733620452574/Put/seqid=0 2024-12-08T01:14:13,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741909_1085 (size=12301) 2024-12-08T01:14:13,653 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/fadcaef66b8346c5a096cd972cb31de2 2024-12-08T01:14:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/b348719eebb34fb8b560091fd1437edf is 50, key is test_row_0/B:col10/1733620452574/Put/seqid=0 2024-12-08T01:14:13,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741910_1086 (size=12301) 2024-12-08T01:14:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T01:14:13,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620453759, max timestamp=1733620453778Num uniques 859; 2024-12-08T01:14:13,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620453759, max timestamp=1733620453778Num uniques 859; 2024-12-08T01:14:13,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:13,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:13,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620453759, max timestamp=1733620453778Num uniques 859; 2024-12-08T01:14:13,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:13,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620513829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620513831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620513944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620513944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T01:14:14,088 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/b348719eebb34fb8b560091fd1437edf 2024-12-08T01:14:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/a5c426e638d04ec88681eab4fde07443 is 50, key is test_row_0/C:col10/1733620452574/Put/seqid=0 2024-12-08T01:14:14,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741911_1087 (size=12301) 2024-12-08T01:14:14,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620514142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,144 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,143 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/a5c426e638d04ec88681eab4fde07443 2024-12-08T01:14:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620514153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620514153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620514153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,161 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/fadcaef66b8346c5a096cd972cb31de2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/fadcaef66b8346c5a096cd972cb31de2 2024-12-08T01:14:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,181 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/fadcaef66b8346c5a096cd972cb31de2, entries=150, sequenceid=354, filesize=12.0 K 2024-12-08T01:14:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/b348719eebb34fb8b560091fd1437edf as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/b348719eebb34fb8b560091fd1437edf 2024-12-08T01:14:14,193 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/b348719eebb34fb8b560091fd1437edf, entries=150, sequenceid=354, filesize=12.0 K 2024-12-08T01:14:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/a5c426e638d04ec88681eab4fde07443 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/a5c426e638d04ec88681eab4fde07443 2024-12-08T01:14:14,202 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/a5c426e638d04ec88681eab4fde07443, entries=150, sequenceid=354, filesize=12.0 K 2024-12-08T01:14:14,203 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13ab931b3a97ca4195bbab8a55597446 in 595ms, sequenceid=354, compaction requested=true 2024-12-08T01:14:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-08T01:14:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-08T01:14:14,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-08T01:14:14,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 750 msec 2024-12-08T01:14:14,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 757 msec 2024-12-08T01:14:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:14,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T01:14:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:14,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/89106903a3784bdb8c08e7ae290605f9 is 50, key is test_row_0/A:col10/1733620453819/Put/seqid=0 2024-12-08T01:14:14,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620514481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620514482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741912_1088 (size=14741) 2024-12-08T01:14:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T01:14:14,592 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:14,593 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-08T01:14:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T01:14:14,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:14,597 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:14,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:14,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620514592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620514592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T01:14:14,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T01:14:14,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:14,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:14,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:14,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:14,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620514803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620514803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:14,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/89106903a3784bdb8c08e7ae290605f9 2024-12-08T01:14:14,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/4d713fc9edfc4c02b9e70071a9c93d97 is 50, key is test_row_0/B:col10/1733620453819/Put/seqid=0 2024-12-08T01:14:14,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T01:14:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:14,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:14,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741913_1089 (size=12301) 2024-12-08T01:14:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:14,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/4d713fc9edfc4c02b9e70071a9c93d97 2024-12-08T01:14:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T01:14:14,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/92eb195ebf71460bb50f657da601dddc is 50, key is test_row_0/C:col10/1733620453819/Put/seqid=0 2024-12-08T01:14:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741914_1090 (size=12301) 2024-12-08T01:14:15,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T01:14:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:15,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620515111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620515114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:15,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T01:14:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T01:14:15,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/92eb195ebf71460bb50f657da601dddc 2024-12-08T01:14:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/89106903a3784bdb8c08e7ae290605f9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/89106903a3784bdb8c08e7ae290605f9 2024-12-08T01:14:15,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/89106903a3784bdb8c08e7ae290605f9, entries=200, sequenceid=391, filesize=14.4 K 2024-12-08T01:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/4d713fc9edfc4c02b9e70071a9c93d97 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4d713fc9edfc4c02b9e70071a9c93d97 2024-12-08T01:14:15,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4d713fc9edfc4c02b9e70071a9c93d97, entries=150, sequenceid=391, filesize=12.0 K 2024-12-08T01:14:15,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/92eb195ebf71460bb50f657da601dddc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/92eb195ebf71460bb50f657da601dddc 2024-12-08T01:14:15,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/92eb195ebf71460bb50f657da601dddc, entries=150, sequenceid=391, filesize=12.0 K 2024-12-08T01:14:15,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 13ab931b3a97ca4195bbab8a55597446 in 879ms, sequenceid=391, compaction requested=true 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:15,347 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:15,347 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:15,348 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:15,349 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:15,349 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:15,349 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:15,349 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,349 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,349 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/585c8b9a942948c38d1ac3bcda71549e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7d0795422ce346d38bf0b0809f9e55f9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/b348719eebb34fb8b560091fd1437edf, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4d713fc9edfc4c02b9e70071a9c93d97] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=48.8 K 2024-12-08T01:14:15,349 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c304cb59ac31464ca47d5ae6a8da18b0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0c33f9e08ba742c496c3b24b54d9f757, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/fadcaef66b8346c5a096cd972cb31de2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/89106903a3784bdb8c08e7ae290605f9] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=51.2 K 2024-12-08T01:14:15,349 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 585c8b9a942948c38d1ac3bcda71549e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733620451831 2024-12-08T01:14:15,349 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c304cb59ac31464ca47d5ae6a8da18b0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733620451831 2024-12-08T01:14:15,350 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0795422ce346d38bf0b0809f9e55f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733620451932 2024-12-08T01:14:15,350 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c33f9e08ba742c496c3b24b54d9f757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733620451932 2024-12-08T01:14:15,350 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b348719eebb34fb8b560091fd1437edf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733620452565 2024-12-08T01:14:15,350 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadcaef66b8346c5a096cd972cb31de2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733620452565 2024-12-08T01:14:15,350 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d713fc9edfc4c02b9e70071a9c93d97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733620453819 2024-12-08T01:14:15,350 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89106903a3784bdb8c08e7ae290605f9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733620453819 2024-12-08T01:14:15,362 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:15,363 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7f8de72f9a364906a4b19ef42bafbfec is 50, key is test_row_0/B:col10/1733620453819/Put/seqid=0 2024-12-08T01:14:15,365 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:15,366 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/efc120d5e1074deeb13afaf91d41ff4e is 50, key is test_row_0/A:col10/1733620453819/Put/seqid=0 2024-12-08T01:14:15,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T01:14:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,368 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-08T01:14:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:15,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/e8b5926cc2144a0d9bcd99b0801a54f4 is 50, key is test_row_0/A:col10/1733620454480/Put/seqid=0 2024-12-08T01:14:15,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741916_1092 (size=13187) 2024-12-08T01:14:15,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741917_1093 (size=12301) 2024-12-08T01:14:15,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741915_1091 (size=13187) 2024-12-08T01:14:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:15,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:15,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620455676, max timestamp=1733620455689Num uniques 859; 2024-12-08T01:14:15,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620455676, max timestamp=1733620455689Num uniques 859; 2024-12-08T01:14:15,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:15,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:15,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620455676, max timestamp=1733620455689Num uniques 859; 2024-12-08T01:14:15,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T01:14:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620515752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620515758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:15,777 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/e8b5926cc2144a0d9bcd99b0801a54f4 2024-12-08T01:14:15,783 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/efc120d5e1074deeb13afaf91d41ff4e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/efc120d5e1074deeb13afaf91d41ff4e 2024-12-08T01:14:15,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7cbf0cfc025a45afac0dfbc04e22903f is 50, key is test_row_0/B:col10/1733620454480/Put/seqid=0 2024-12-08T01:14:15,792 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7f8de72f9a364906a4b19ef42bafbfec as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7f8de72f9a364906a4b19ef42bafbfec 2024-12-08T01:14:15,798 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into efc120d5e1074deeb13afaf91d41ff4e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:15,798 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:15,798 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=12, startTime=1733620455347; duration=0sec 2024-12-08T01:14:15,798 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:15,798 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:15,799 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:15,802 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:15,802 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:15,803 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:15,803 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b97ef089ab6a45d1853384ba5d0bfda5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fa6ad2f57a744011bac9f8543241cb77, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/a5c426e638d04ec88681eab4fde07443, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/92eb195ebf71460bb50f657da601dddc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=48.8 K 2024-12-08T01:14:15,804 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 7f8de72f9a364906a4b19ef42bafbfec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:15,804 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97ef089ab6a45d1853384ba5d0bfda5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733620451831 2024-12-08T01:14:15,804 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:15,804 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=12, startTime=1733620455347; duration=0sec 2024-12-08T01:14:15,804 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:15,804 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:15,805 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa6ad2f57a744011bac9f8543241cb77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733620451932 2024-12-08T01:14:15,805 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c426e638d04ec88681eab4fde07443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733620452565 2024-12-08T01:14:15,806 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92eb195ebf71460bb50f657da601dddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733620453819 2024-12-08T01:14:15,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741918_1094 (size=12301) 2024-12-08T01:14:15,823 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:15,824 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/7cdc7c32d7804dd3a79d51b257711f5b is 50, key is test_row_0/C:col10/1733620453819/Put/seqid=0 2024-12-08T01:14:15,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741919_1095 (size=13187) 2024-12-08T01:14:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620515862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:15,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620515872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620516071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620516082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620516162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,168 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620516172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,176 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,221 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7cbf0cfc025a45afac0dfbc04e22903f 2024-12-08T01:14:16,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/62b2958593fc40e9aabca69a337251f3 is 50, key is test_row_0/C:col10/1733620454480/Put/seqid=0 2024-12-08T01:14:16,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741920_1096 (size=12301) 2024-12-08T01:14:16,238 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/62b2958593fc40e9aabca69a337251f3 2024-12-08T01:14:16,240 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/7cdc7c32d7804dd3a79d51b257711f5b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/7cdc7c32d7804dd3a79d51b257711f5b 2024-12-08T01:14:16,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/e8b5926cc2144a0d9bcd99b0801a54f4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/e8b5926cc2144a0d9bcd99b0801a54f4 2024-12-08T01:14:16,250 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 7cdc7c32d7804dd3a79d51b257711f5b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:16,250 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:16,251 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=12, startTime=1733620455347; duration=0sec 2024-12-08T01:14:16,251 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:16,251 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:16,254 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/e8b5926cc2144a0d9bcd99b0801a54f4, entries=150, sequenceid=400, filesize=12.0 K 2024-12-08T01:14:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/7cbf0cfc025a45afac0dfbc04e22903f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7cbf0cfc025a45afac0dfbc04e22903f 2024-12-08T01:14:16,262 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7cbf0cfc025a45afac0dfbc04e22903f, entries=150, sequenceid=400, filesize=12.0 K 2024-12-08T01:14:16,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/62b2958593fc40e9aabca69a337251f3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/62b2958593fc40e9aabca69a337251f3 2024-12-08T01:14:16,270 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/62b2958593fc40e9aabca69a337251f3, entries=150, sequenceid=400, filesize=12.0 K 2024-12-08T01:14:16,271 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 13ab931b3a97ca4195bbab8a55597446 in 904ms, sequenceid=400, compaction requested=false 2024-12-08T01:14:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-08T01:14:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-08T01:14:16,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6830 sec 2024-12-08T01:14:16,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T01:14:16,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6760 sec 2024-12-08T01:14:16,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733620455727, max timestamp=1733620456384Num uniques 1068; 2024-12-08T01:14:16,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T01:14:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:16,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=107.34 KB heapSize=302.36 KB 2024-12-08T01:14:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:16,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:16,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/389bad7d4ec04f8189e233f563e2b5a0 is 50, key is test_row_0/A:col10/1733620455749/Put/seqid=0 2024-12-08T01:14:16,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741921_1097 (size=14741) 2024-12-08T01:14:16,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/389bad7d4ec04f8189e233f563e2b5a0 2024-12-08T01:14:16,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/49c22fd1124749c9b5791b8ff6ba5b4e is 50, key is test_row_0/B:col10/1733620455749/Put/seqid=0 2024-12-08T01:14:16,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620516428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620516429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741922_1098 (size=12301) 2024-12-08T01:14:16,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/49c22fd1124749c9b5791b8ff6ba5b4e 2024-12-08T01:14:16,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/2523299fce9648ee8353cbaacf2050e7 is 50, key is test_row_0/C:col10/1733620455749/Put/seqid=0 2024-12-08T01:14:16,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741923_1099 (size=12301) 2024-12-08T01:14:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620516542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620516542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T01:14:16,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:16,743 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-08T01:14:16,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T01:14:16,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:16,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:16,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620516751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620516751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T01:14:16,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/2523299fce9648ee8353cbaacf2050e7 2024-12-08T01:14:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/389bad7d4ec04f8189e233f563e2b5a0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/389bad7d4ec04f8189e233f563e2b5a0 2024-12-08T01:14:16,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/389bad7d4ec04f8189e233f563e2b5a0, entries=200, sequenceid=441, filesize=14.4 K 2024-12-08T01:14:16,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/49c22fd1124749c9b5791b8ff6ba5b4e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/49c22fd1124749c9b5791b8ff6ba5b4e 2024-12-08T01:14:16,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T01:14:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:16,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:16,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/49c22fd1124749c9b5791b8ff6ba5b4e, entries=150, sequenceid=441, filesize=12.0 K 2024-12-08T01:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:16,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/2523299fce9648ee8353cbaacf2050e7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/2523299fce9648ee8353cbaacf2050e7 2024-12-08T01:14:16,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/2523299fce9648ee8353cbaacf2050e7, entries=150, sequenceid=441, filesize=12.0 K 2024-12-08T01:14:16,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~302.31 KB/309568, currentSize=80.51 KB/82440 for 13ab931b3a97ca4195bbab8a55597446 in 521ms, sequenceid=441, compaction requested=true 2024-12-08T01:14:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:16,909 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:16,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:16,910 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:16,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:16,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:16,911 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:16,911 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:16,911 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:16,911 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/efc120d5e1074deeb13afaf91d41ff4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/e8b5926cc2144a0d9bcd99b0801a54f4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/389bad7d4ec04f8189e233f563e2b5a0] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=39.3 K 2024-12-08T01:14:16,911 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:16,911 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:16,911 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:16,911 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7f8de72f9a364906a4b19ef42bafbfec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7cbf0cfc025a45afac0dfbc04e22903f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/49c22fd1124749c9b5791b8ff6ba5b4e] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=36.9 K 2024-12-08T01:14:16,913 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc120d5e1074deeb13afaf91d41ff4e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733620453819 2024-12-08T01:14:16,913 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8de72f9a364906a4b19ef42bafbfec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733620453819 2024-12-08T01:14:16,913 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b5926cc2144a0d9bcd99b0801a54f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733620454475 2024-12-08T01:14:16,913 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cbf0cfc025a45afac0dfbc04e22903f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733620454475 2024-12-08T01:14:16,913 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 49c22fd1124749c9b5791b8ff6ba5b4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733620455727 2024-12-08T01:14:16,914 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389bad7d4ec04f8189e233f563e2b5a0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733620455727 2024-12-08T01:14:16,927 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:16,928 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/021ad5e4d90a4d57a7e1a423ca2c0733 is 50, key is test_row_0/B:col10/1733620455749/Put/seqid=0 2024-12-08T01:14:16,931 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:16,932 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/27e2f6bca969499a9e5fa9831720cba7 is 50, key is test_row_0/A:col10/1733620455749/Put/seqid=0 2024-12-08T01:14:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741925_1101 (size=13289) 2024-12-08T01:14:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741924_1100 (size=13289) 2024-12-08T01:14:16,945 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/27e2f6bca969499a9e5fa9831720cba7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/27e2f6bca969499a9e5fa9831720cba7 2024-12-08T01:14:16,952 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into 27e2f6bca969499a9e5fa9831720cba7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:16,952 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:16,952 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620456909; duration=0sec 2024-12-08T01:14:16,952 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:16,952 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:16,952 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:16,954 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:16,954 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:16,954 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:16,954 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/7cdc7c32d7804dd3a79d51b257711f5b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/62b2958593fc40e9aabca69a337251f3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/2523299fce9648ee8353cbaacf2050e7] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=36.9 K 2024-12-08T01:14:16,955 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cdc7c32d7804dd3a79d51b257711f5b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733620453819 2024-12-08T01:14:16,955 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b2958593fc40e9aabca69a337251f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733620454475 2024-12-08T01:14:16,956 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2523299fce9648ee8353cbaacf2050e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733620455727 2024-12-08T01:14:16,965 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:16,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/28d9bcb13e1c49eca39b8bac89854571 is 50, key is test_row_0/C:col10/1733620455749/Put/seqid=0 2024-12-08T01:14:16,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741926_1102 (size=13289) 2024-12-08T01:14:16,984 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/28d9bcb13e1c49eca39b8bac89854571 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/28d9bcb13e1c49eca39b8bac89854571 2024-12-08T01:14:16,990 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 28d9bcb13e1c49eca39b8bac89854571(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:16,990 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:16,990 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620456910; duration=0sec 2024-12-08T01:14:16,991 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:16,991 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:17,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T01:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:17,053 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:17,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/0fc9f674cdba4d3498cedd5b14a8398a is 50, key is test_row_0/A:col10/1733620456428/Put/seqid=0 2024-12-08T01:14:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T01:14:17,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741927_1103 (size=12301) 2024-12-08T01:14:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:17,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:17,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620517098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620517100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620517212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620517212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,345 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/021ad5e4d90a4d57a7e1a423ca2c0733 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/021ad5e4d90a4d57a7e1a423ca2c0733 2024-12-08T01:14:17,354 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into 021ad5e4d90a4d57a7e1a423ca2c0733(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:17,355 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:17,355 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620456909; duration=0sec 2024-12-08T01:14:17,355 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:17,355 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T01:14:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620517422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620517422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,463 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/0fc9f674cdba4d3498cedd5b14a8398a 2024-12-08T01:14:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/156d9fc5278b4af1bfe36ca3f7c778a9 is 50, key is test_row_0/B:col10/1733620456428/Put/seqid=0 2024-12-08T01:14:17,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741928_1104 (size=12301) 2024-12-08T01:14:17,491 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/156d9fc5278b4af1bfe36ca3f7c778a9 2024-12-08T01:14:17,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/41959547d7924c949c7170e84b4785a0 is 50, key is test_row_0/C:col10/1733620456428/Put/seqid=0 2024-12-08T01:14:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741929_1105 (size=12301) 2024-12-08T01:14:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620517732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620517732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T01:14:17,923 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/41959547d7924c949c7170e84b4785a0 2024-12-08T01:14:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/0fc9f674cdba4d3498cedd5b14a8398a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0fc9f674cdba4d3498cedd5b14a8398a 2024-12-08T01:14:17,934 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0fc9f674cdba4d3498cedd5b14a8398a, entries=150, sequenceid=458, filesize=12.0 K 2024-12-08T01:14:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/156d9fc5278b4af1bfe36ca3f7c778a9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/156d9fc5278b4af1bfe36ca3f7c778a9 2024-12-08T01:14:17,940 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/156d9fc5278b4af1bfe36ca3f7c778a9, entries=150, sequenceid=458, filesize=12.0 K 2024-12-08T01:14:17,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/41959547d7924c949c7170e84b4785a0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/41959547d7924c949c7170e84b4785a0 2024-12-08T01:14:17,946 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/41959547d7924c949c7170e84b4785a0, entries=150, sequenceid=458, filesize=12.0 K 2024-12-08T01:14:17,947 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 13ab931b3a97ca4195bbab8a55597446 in 894ms, sequenceid=458, compaction requested=false 2024-12-08T01:14:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-08T01:14:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-08T01:14:17,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T01:14:17,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2030 sec 2024-12-08T01:14:17,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.2080 sec 2024-12-08T01:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:18,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:14:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:18,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6c970ec487e44a68a6db793b5a5b046d is 50, key is test_row_0/A:col10/1733620457097/Put/seqid=0 2024-12-08T01:14:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620518270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620518271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741930_1106 (size=12301) 2024-12-08T01:14:18,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6c970ec487e44a68a6db793b5a5b046d 2024-12-08T01:14:18,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/3cc59dad62344e45a9080c427e49b4b1 is 50, key is test_row_0/B:col10/1733620457097/Put/seqid=0 2024-12-08T01:14:18,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741931_1107 (size=12301) 2024-12-08T01:14:18,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620518382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620518382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620518592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620518592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/3cc59dad62344e45a9080c427e49b4b1 2024-12-08T01:14:18,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b26fe07b1f3c4628b0d3de45737c5f48 is 50, key is test_row_0/C:col10/1733620457097/Put/seqid=0 2024-12-08T01:14:18,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741932_1108 (size=12301) 2024-12-08T01:14:18,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b26fe07b1f3c4628b0d3de45737c5f48 2024-12-08T01:14:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6c970ec487e44a68a6db793b5a5b046d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c970ec487e44a68a6db793b5a5b046d 2024-12-08T01:14:18,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c970ec487e44a68a6db793b5a5b046d, entries=150, sequenceid=478, filesize=12.0 K 2024-12-08T01:14:18,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/3cc59dad62344e45a9080c427e49b4b1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/3cc59dad62344e45a9080c427e49b4b1 2024-12-08T01:14:18,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/3cc59dad62344e45a9080c427e49b4b1, entries=150, sequenceid=478, filesize=12.0 K 2024-12-08T01:14:18,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/b26fe07b1f3c4628b0d3de45737c5f48 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b26fe07b1f3c4628b0d3de45737c5f48 2024-12-08T01:14:18,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b26fe07b1f3c4628b0d3de45737c5f48, entries=150, sequenceid=478, filesize=12.0 K 2024-12-08T01:14:18,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 13ab931b3a97ca4195bbab8a55597446 in 533ms, sequenceid=478, compaction requested=true 2024-12-08T01:14:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:18,777 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:18,777 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:18,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:18,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:18,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:18,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:18,780 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:18,780 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:18,780 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:18,780 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:18,780 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:18,780 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:18,780 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/021ad5e4d90a4d57a7e1a423ca2c0733, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/156d9fc5278b4af1bfe36ca3f7c778a9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/3cc59dad62344e45a9080c427e49b4b1] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.0 K 2024-12-08T01:14:18,780 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/27e2f6bca969499a9e5fa9831720cba7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0fc9f674cdba4d3498cedd5b14a8398a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c970ec487e44a68a6db793b5a5b046d] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.0 K 2024-12-08T01:14:18,781 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 021ad5e4d90a4d57a7e1a423ca2c0733, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733620455727 2024-12-08T01:14:18,781 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e2f6bca969499a9e5fa9831720cba7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733620455727 2024-12-08T01:14:18,781 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 156d9fc5278b4af1bfe36ca3f7c778a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733620456420 2024-12-08T01:14:18,781 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc9f674cdba4d3498cedd5b14a8398a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733620456420 2024-12-08T01:14:18,781 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc59dad62344e45a9080c427e49b4b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733620457074 2024-12-08T01:14:18,781 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c970ec487e44a68a6db793b5a5b046d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733620457074 2024-12-08T01:14:18,790 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:18,791 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2ab33b8466eb4549a3dca561d9b3f0a6 is 50, key is test_row_0/A:col10/1733620457097/Put/seqid=0 2024-12-08T01:14:18,791 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:18,791 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/be39c3ba70604481a9ef74eca547d277 is 50, key is test_row_0/B:col10/1733620457097/Put/seqid=0 2024-12-08T01:14:18,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741934_1110 (size=13391) 2024-12-08T01:14:18,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741933_1109 (size=13391) 2024-12-08T01:14:18,808 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/be39c3ba70604481a9ef74eca547d277 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/be39c3ba70604481a9ef74eca547d277 2024-12-08T01:14:18,812 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2ab33b8466eb4549a3dca561d9b3f0a6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2ab33b8466eb4549a3dca561d9b3f0a6 2024-12-08T01:14:18,817 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into be39c3ba70604481a9ef74eca547d277(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:18,817 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:18,817 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620458777; duration=0sec 2024-12-08T01:14:18,817 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:18,817 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:18,817 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:18,818 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:18,818 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:18,819 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:18,819 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/28d9bcb13e1c49eca39b8bac89854571, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/41959547d7924c949c7170e84b4785a0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b26fe07b1f3c4628b0d3de45737c5f48] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.0 K 2024-12-08T01:14:18,819 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d9bcb13e1c49eca39b8bac89854571, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733620455727 2024-12-08T01:14:18,820 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 41959547d7924c949c7170e84b4785a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733620456420 2024-12-08T01:14:18,820 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into 2ab33b8466eb4549a3dca561d9b3f0a6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:18,820 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:18,820 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620458777; duration=0sec 2024-12-08T01:14:18,820 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b26fe07b1f3c4628b0d3de45737c5f48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733620457074 2024-12-08T01:14:18,820 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:18,820 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:18,830 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:18,831 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/d6d3ee97e464492e998fea422d085d5f is 50, key is test_row_0/C:col10/1733620457097/Put/seqid=0 2024-12-08T01:14:18,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741935_1111 (size=13391) 2024-12-08T01:14:18,849 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/d6d3ee97e464492e998fea422d085d5f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/d6d3ee97e464492e998fea422d085d5f 2024-12-08T01:14:18,858 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into d6d3ee97e464492e998fea422d085d5f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:18,858 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:18,858 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620458780; duration=0sec 2024-12-08T01:14:18,858 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:18,858 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T01:14:18,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:18,893 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-08T01:14:18,895 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:18,897 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:18,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:18,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:18,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/148518bc08a541e38637d83189dc77f2 is 50, key is test_row_0/A:col10/1733620458269/Put/seqid=0 2024-12-08T01:14:18,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741936_1112 (size=17181) 2024-12-08T01:14:18,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620518933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:18,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620518934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620519042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620519042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620519252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:19,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620519252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/148518bc08a541e38637d83189dc77f2 2024-12-08T01:14:19,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/925fb79ad0ae469f8399a372d1a633cd is 50, key is test_row_0/B:col10/1733620458269/Put/seqid=0 2024-12-08T01:14:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741937_1113 (size=12301) 2024-12-08T01:14:19,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:19,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620519562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620519562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:19,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/925fb79ad0ae469f8399a372d1a633cd 2024-12-08T01:14:19,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/526f3df993334cd78d59aa266799d85c is 50, key is test_row_0/C:col10/1733620458269/Put/seqid=0 2024-12-08T01:14:19,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741938_1114 (size=12301) 2024-12-08T01:14:19,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:19,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620520072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620520072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/526f3df993334cd78d59aa266799d85c 2024-12-08T01:14:20,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/148518bc08a541e38637d83189dc77f2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/148518bc08a541e38637d83189dc77f2 2024-12-08T01:14:20,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/148518bc08a541e38637d83189dc77f2, entries=250, sequenceid=496, filesize=16.8 K 2024-12-08T01:14:20,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/925fb79ad0ae469f8399a372d1a633cd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/925fb79ad0ae469f8399a372d1a633cd 2024-12-08T01:14:20,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/925fb79ad0ae469f8399a372d1a633cd, entries=150, sequenceid=496, filesize=12.0 K 2024-12-08T01:14:20,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/526f3df993334cd78d59aa266799d85c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/526f3df993334cd78d59aa266799d85c 2024-12-08T01:14:20,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/526f3df993334cd78d59aa266799d85c, entries=150, sequenceid=496, filesize=12.0 K 2024-12-08T01:14:20,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 13ab931b3a97ca4195bbab8a55597446 in 1278ms, sequenceid=496, compaction requested=false 2024-12-08T01:14:20,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:20,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:14:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:20,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2d81f97527d2420a9aad67a52b661055 is 50, key is test_row_0/A:col10/1733620458932/Put/seqid=0 2024-12-08T01:14:20,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741939_1115 (size=14741) 2024-12-08T01:14:20,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620520218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620520222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620520332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620520332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620520542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620520542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2d81f97527d2420a9aad67a52b661055 2024-12-08T01:14:20,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/0b948ea9e01b4185894d8892f62d6c18 is 50, key is test_row_0/B:col10/1733620458932/Put/seqid=0 2024-12-08T01:14:20,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741940_1116 (size=12301) 2024-12-08T01:14:20,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:20,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:20,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620520866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620520867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:20,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:20,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:20,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:20,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/0b948ea9e01b4185894d8892f62d6c18 2024-12-08T01:14:21,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/de8d153cfcad422db468d25915f9fafb is 50, key is test_row_0/C:col10/1733620458932/Put/seqid=0 2024-12-08T01:14:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741941_1117 (size=12301) 2024-12-08T01:14:21,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/de8d153cfcad422db468d25915f9fafb 2024-12-08T01:14:21,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/2d81f97527d2420a9aad67a52b661055 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2d81f97527d2420a9aad67a52b661055 2024-12-08T01:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:21,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2d81f97527d2420a9aad67a52b661055, entries=200, sequenceid=514, filesize=14.4 K 2024-12-08T01:14:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/0b948ea9e01b4185894d8892f62d6c18 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0b948ea9e01b4185894d8892f62d6c18 2024-12-08T01:14:21,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:21,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0b948ea9e01b4185894d8892f62d6c18, entries=150, sequenceid=514, filesize=12.0 K 2024-12-08T01:14:21,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/de8d153cfcad422db468d25915f9fafb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/de8d153cfcad422db468d25915f9fafb 2024-12-08T01:14:21,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/de8d153cfcad422db468d25915f9fafb, entries=150, sequenceid=514, filesize=12.0 K 2024-12-08T01:14:21,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13ab931b3a97ca4195bbab8a55597446 in 870ms, sequenceid=514, compaction requested=true 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:21,063 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:21,063 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13ab931b3a97ca4195bbab8a55597446:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:21,064 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:21,064 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:21,064 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/A is initiating minor compaction (all files) 2024-12-08T01:14:21,064 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/B is initiating minor compaction (all files) 2024-12-08T01:14:21,064 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/A in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,064 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/B in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,064 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2ab33b8466eb4549a3dca561d9b3f0a6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/148518bc08a541e38637d83189dc77f2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2d81f97527d2420a9aad67a52b661055] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=44.3 K 2024-12-08T01:14:21,064 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/be39c3ba70604481a9ef74eca547d277, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/925fb79ad0ae469f8399a372d1a633cd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0b948ea9e01b4185894d8892f62d6c18] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.1 K 2024-12-08T01:14:21,065 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting be39c3ba70604481a9ef74eca547d277, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733620457074 2024-12-08T01:14:21,065 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 925fb79ad0ae469f8399a372d1a633cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733620458269 2024-12-08T01:14:21,065 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab33b8466eb4549a3dca561d9b3f0a6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733620457074 2024-12-08T01:14:21,065 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b948ea9e01b4185894d8892f62d6c18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1733620458929 2024-12-08T01:14:21,065 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 148518bc08a541e38637d83189dc77f2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733620458259 2024-12-08T01:14:21,066 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d81f97527d2420a9aad67a52b661055, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1733620458925 2024-12-08T01:14:21,071 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:21,072 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/8ad0656c103d4f82a5840d6f33a8792f is 50, key is test_row_0/A:col10/1733620458932/Put/seqid=0 2024-12-08T01:14:21,072 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:21,072 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/d358f7313dd7485480541bca47464c9b is 50, key is test_row_0/B:col10/1733620458932/Put/seqid=0 2024-12-08T01:14:21,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741942_1118 (size=13493) 2024-12-08T01:14:21,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:21,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:21,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:21,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:21,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:21,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:21,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:21,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741943_1119 (size=13493) 2024-12-08T01:14:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:21,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/5a48a1bf61f94ccb8da96739a23cce78 is 50, key is test_row_0/A:col10/1733620461083/Put/seqid=0 2024-12-08T01:14:21,092 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/d358f7313dd7485480541bca47464c9b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/d358f7313dd7485480541bca47464c9b 2024-12-08T01:14:21,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741944_1120 (size=12301) 2024-12-08T01:14:21,099 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/B of 13ab931b3a97ca4195bbab8a55597446 into d358f7313dd7485480541bca47464c9b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:21,099 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:21,099 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/B, priority=13, startTime=1733620461063; duration=0sec 2024-12-08T01:14:21,099 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:21,099 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:B 2024-12-08T01:14:21,100 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:21,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/5a48a1bf61f94ccb8da96739a23cce78 2024-12-08T01:14:21,105 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:21,105 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 13ab931b3a97ca4195bbab8a55597446/C is initiating minor compaction (all files) 2024-12-08T01:14:21,106 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13ab931b3a97ca4195bbab8a55597446/C in TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,106 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/d6d3ee97e464492e998fea422d085d5f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/526f3df993334cd78d59aa266799d85c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/de8d153cfcad422db468d25915f9fafb] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp, totalSize=37.1 K 2024-12-08T01:14:21,108 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d6d3ee97e464492e998fea422d085d5f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733620457074 2024-12-08T01:14:21,108 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 526f3df993334cd78d59aa266799d85c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733620458269 2024-12-08T01:14:21,109 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting de8d153cfcad422db468d25915f9fafb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1733620458929 2024-12-08T01:14:21,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/38b00f9f78384fdab8ee3b9f41efea3a is 50, key is test_row_0/B:col10/1733620461083/Put/seqid=0 2024-12-08T01:14:21,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741945_1121 (size=12301) 2024-12-08T01:14:21,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/38b00f9f78384fdab8ee3b9f41efea3a 2024-12-08T01:14:21,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620521122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620521126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,131 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13ab931b3a97ca4195bbab8a55597446#C#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,132 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/417c03c3bd4b47039950749374a1741a is 50, key is test_row_0/C:col10/1733620458932/Put/seqid=0 2024-12-08T01:14:21,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/560a34d173ec410fa68451bdcb79544a is 50, key is test_row_0/C:col10/1733620461083/Put/seqid=0 2024-12-08T01:14:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741946_1122 (size=13493) 2024-12-08T01:14:21,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741947_1123 (size=12301) 2024-12-08T01:14:21,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:21,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620521232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620521242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620521382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620521382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620521442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620521452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,482 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/8ad0656c103d4f82a5840d6f33a8792f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8ad0656c103d4f82a5840d6f33a8792f 2024-12-08T01:14:21,493 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/A of 13ab931b3a97ca4195bbab8a55597446 into 8ad0656c103d4f82a5840d6f33a8792f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:21,493 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:21,493 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/A, priority=13, startTime=1733620461063; duration=0sec 2024-12-08T01:14:21,494 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:21,494 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:A 2024-12-08T01:14:21,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:21,552 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/417c03c3bd4b47039950749374a1741a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/417c03c3bd4b47039950749374a1741a 2024-12-08T01:14:21,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/560a34d173ec410fa68451bdcb79544a 2024-12-08T01:14:21,562 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13ab931b3a97ca4195bbab8a55597446/C of 13ab931b3a97ca4195bbab8a55597446 into 417c03c3bd4b47039950749374a1741a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:21,562 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:21,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/5a48a1bf61f94ccb8da96739a23cce78 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/5a48a1bf61f94ccb8da96739a23cce78 2024-12-08T01:14:21,562 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., storeName=13ab931b3a97ca4195bbab8a55597446/C, priority=13, startTime=1733620461063; duration=0sec 2024-12-08T01:14:21,562 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:21,562 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13ab931b3a97ca4195bbab8a55597446:C 2024-12-08T01:14:21,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/5a48a1bf61f94ccb8da96739a23cce78, entries=150, sequenceid=529, filesize=12.0 K 2024-12-08T01:14:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/38b00f9f78384fdab8ee3b9f41efea3a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/38b00f9f78384fdab8ee3b9f41efea3a 2024-12-08T01:14:21,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/38b00f9f78384fdab8ee3b9f41efea3a, entries=150, sequenceid=529, filesize=12.0 K 2024-12-08T01:14:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/560a34d173ec410fa68451bdcb79544a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/560a34d173ec410fa68451bdcb79544a 2024-12-08T01:14:21,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/560a34d173ec410fa68451bdcb79544a, entries=150, sequenceid=529, filesize=12.0 K 2024-12-08T01:14:21,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 13ab931b3a97ca4195bbab8a55597446 in 501ms, sequenceid=529, compaction requested=false 2024-12-08T01:14:21,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:21,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-08T01:14:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:21,663 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:14:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:21,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/64147abfb786445cbedaaaad40ce60fe is 50, key is test_row_0/A:col10/1733620461121/Put/seqid=0 2024-12-08T01:14:21,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741948_1124 (size=12301) 2024-12-08T01:14:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:21,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620521775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620521776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620521882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620521882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,085 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/64147abfb786445cbedaaaad40ce60fe 2024-12-08T01:14:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620522092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620522096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/6706b369a16d4b099d219dd14a04b7e9 is 50, key is test_row_0/B:col10/1733620461121/Put/seqid=0 2024-12-08T01:14:22,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741949_1125 (size=12301) 2024-12-08T01:14:22,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40328 deadline: 1733620522392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1733620522392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,396 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:22,396 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620522401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620522401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,520 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/6706b369a16d4b099d219dd14a04b7e9 2024-12-08T01:14:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/ad4128dae7434e9599fb3be3d21648e9 is 50, key is test_row_0/C:col10/1733620461121/Put/seqid=0 2024-12-08T01:14:22,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741950_1126 (size=12301) 2024-12-08T01:14:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40344 deadline: 1733620522651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,652 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 13ab931b3a97ca4195bbab8a55597446 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18325 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40352 deadline: 1733620522911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40316 deadline: 1733620522911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 2024-12-08T01:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:14:22,939 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/ad4128dae7434e9599fb3be3d21648e9 2024-12-08T01:14:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/64147abfb786445cbedaaaad40ce60fe as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/64147abfb786445cbedaaaad40ce60fe 2024-12-08T01:14:22,948 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/64147abfb786445cbedaaaad40ce60fe, entries=150, sequenceid=549, filesize=12.0 K 2024-12-08T01:14:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/6706b369a16d4b099d219dd14a04b7e9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/6706b369a16d4b099d219dd14a04b7e9 2024-12-08T01:14:22,954 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/6706b369a16d4b099d219dd14a04b7e9, entries=150, sequenceid=549, filesize=12.0 K 2024-12-08T01:14:22,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/ad4128dae7434e9599fb3be3d21648e9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/ad4128dae7434e9599fb3be3d21648e9 2024-12-08T01:14:22,960 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/ad4128dae7434e9599fb3be3d21648e9, entries=150, sequenceid=549, filesize=12.0 K 2024-12-08T01:14:22,961 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 13ab931b3a97ca4195bbab8a55597446 in 1298ms, sequenceid=549, compaction requested=true 2024-12-08T01:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-08T01:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-08T01:14:22,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-08T01:14:22,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0660 sec 2024-12-08T01:14:22,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 4.0740 sec 2024-12-08T01:14:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T01:14:23,062 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:23,063 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-12-08T01:14:23,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-08T01:14:23,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:23,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-08T01:14:23,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:23,221 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:23,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6c3ca7544c214a058df703f8320721ff is 50, key is test_row_0/A:col10/1733620461774/Put/seqid=0 2024-12-08T01:14:23,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741951_1127 (size=12301) 2024-12-08T01:14:23,230 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6c3ca7544c214a058df703f8320721ff 2024-12-08T01:14:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/e290a2c340c442d3a9aef080d2baf1d4 is 50, key is test_row_0/B:col10/1733620461774/Put/seqid=0 2024-12-08T01:14:23,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741952_1128 (size=12301) 2024-12-08T01:14:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-08T01:14:23,414 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-08T01:14:23,414 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:23,416 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-08T01:14:23,416 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-08T01:14:23,416 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:23,416 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:23,417 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-08T01:14:23,417 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:23,419 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:23,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:23,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:23,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:23,642 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/e290a2c340c442d3a9aef080d2baf1d4 2024-12-08T01:14:23,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/eff640b1eac044078b5b86c8719692e4 is 50, key is test_row_0/C:col10/1733620461774/Put/seqid=0 2024-12-08T01:14:23,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741953_1129 (size=12301) 2024-12-08T01:14:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-08T01:14:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:23,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. as already flushing 2024-12-08T01:14:23,927 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-08T01:14:23,928 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:23,928 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,928 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-08T01:14:23,928 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,928 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:23,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:23,928 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,928 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:23,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:24,057 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/eff640b1eac044078b5b86c8719692e4 2024-12-08T01:14:24,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/6c3ca7544c214a058df703f8320721ff as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c3ca7544c214a058df703f8320721ff 2024-12-08T01:14:24,076 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c3ca7544c214a058df703f8320721ff, entries=150, sequenceid=564, filesize=12.0 K 2024-12-08T01:14:24,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/e290a2c340c442d3a9aef080d2baf1d4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/e290a2c340c442d3a9aef080d2baf1d4 2024-12-08T01:14:24,082 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/e290a2c340c442d3a9aef080d2baf1d4, entries=150, sequenceid=564, filesize=12.0 K 2024-12-08T01:14:24,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/eff640b1eac044078b5b86c8719692e4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/eff640b1eac044078b5b86c8719692e4 2024-12-08T01:14:24,087 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/eff640b1eac044078b5b86c8719692e4, entries=150, sequenceid=564, filesize=12.0 K 2024-12-08T01:14:24,088 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 13ab931b3a97ca4195bbab8a55597446 in 867ms, sequenceid=564, compaction requested=true 2024-12-08T01:14:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 13ab931b3a97ca4195bbab8a55597446: 2024-12-08T01:14:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-08T01:14:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-08T01:14:24,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T01:14:24,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-12-08T01:14:24,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.0300 sec 2024-12-08T01:14:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-08T01:14:24,203 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:24,417 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-08T01:14:24,417 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-08T01:14:24,417 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:24,417 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:24,417 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:24,417 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:24,417 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:24,417 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:24,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:24,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:24,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:14:32,707 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-08T01:14:32,707 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:32,707 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:32,708 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:32,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:32,708 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3639 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3536 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1669 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5001 rows 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1676 2024-12-08T01:14:32,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5020 rows 2024-12-08T01:14:32,709 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:14:32,709 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:14:32,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:32,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:32,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:32,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T01:14:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T01:14:32,720 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620472719"}]},"ts":"1733620472719"} 2024-12-08T01:14:32,722 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T01:14:32,722 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T01:14:32,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:14:32,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, UNASSIGN}] 2024-12-08T01:14:32,728 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, UNASSIGN 2024-12-08T01:14:32,729 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=13ab931b3a97ca4195bbab8a55597446, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:32,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, UNASSIGN because future has completed 2024-12-08T01:14:32,732 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:14:32,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630}] 2024-12-08T01:14:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T01:14:32,890 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:32,891 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:14:32,892 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 13ab931b3a97ca4195bbab8a55597446, disabling compactions & flushes 2024-12-08T01:14:32,892 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:32,892 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:32,892 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. after waiting 0 ms 2024-12-08T01:14:32,892 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:32,893 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 13ab931b3a97ca4195bbab8a55597446 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T01:14:32,893 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=A 2024-12-08T01:14:32,893 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:32,893 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=B 2024-12-08T01:14:32,894 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:32,894 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13ab931b3a97ca4195bbab8a55597446, store=C 2024-12-08T01:14:32,894 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:32,900 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/54ada86d855e4430aa8082968e781de7 is 50, key is test_row_0/A:col10/1733620472703/Put/seqid=0 2024-12-08T01:14:32,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741954_1130 (size=9857) 2024-12-08T01:14:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T01:14:33,306 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/54ada86d855e4430aa8082968e781de7 2024-12-08T01:14:33,320 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/69fc0e1ff29745228913abe18d3a1eae is 50, key is test_row_0/B:col10/1733620472703/Put/seqid=0 2024-12-08T01:14:33,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741955_1131 (size=9857) 2024-12-08T01:14:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T01:14:33,726 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/69fc0e1ff29745228913abe18d3a1eae 2024-12-08T01:14:33,735 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/15d1ccbbda1f41948ad8bdd8d0d78632 is 50, key is test_row_0/C:col10/1733620472703/Put/seqid=0 2024-12-08T01:14:33,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741956_1132 (size=9857) 2024-12-08T01:14:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T01:14:34,141 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/15d1ccbbda1f41948ad8bdd8d0d78632 2024-12-08T01:14:34,152 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/A/54ada86d855e4430aa8082968e781de7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/54ada86d855e4430aa8082968e781de7 2024-12-08T01:14:34,157 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/54ada86d855e4430aa8082968e781de7, entries=100, sequenceid=572, filesize=9.6 K 2024-12-08T01:14:34,158 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/B/69fc0e1ff29745228913abe18d3a1eae as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/69fc0e1ff29745228913abe18d3a1eae 2024-12-08T01:14:34,163 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/69fc0e1ff29745228913abe18d3a1eae, entries=100, sequenceid=572, filesize=9.6 K 2024-12-08T01:14:34,164 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/.tmp/C/15d1ccbbda1f41948ad8bdd8d0d78632 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/15d1ccbbda1f41948ad8bdd8d0d78632 2024-12-08T01:14:34,169 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/15d1ccbbda1f41948ad8bdd8d0d78632, entries=100, sequenceid=572, filesize=9.6 K 2024-12-08T01:14:34,170 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 13ab931b3a97ca4195bbab8a55597446 in 1278ms, sequenceid=572, compaction requested=true 2024-12-08T01:14:34,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/746f05f7e3ac474f9ed75293dde7ac74, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a8d8769aaac1450bb6df6308919993df, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a1a9db5f9e494238a91fa3c0f4ddcbcc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/d4c8f44ba1c24380ba4b09cee58d72ad, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/518663db9e144324b1738651337f481e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2164eaf1d4344f7cb2f07a801f084303, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/f5fb77854ee04c71a1f8589542ba210a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/4a8305da48154ad4b0356f366137a396, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/47a0e2f908c44de8bae74a0691b5b547, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/7dbd973bd15448049944e96cf7cef912, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a35eecb111b449259d2c2da1594d2f7c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/08a178f1f7e541a3bda751bc883507bd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/785469833ad245bfa84557d18351e4f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/1fed5dfc0d9e4474be3ae34b949a159b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c94f364efc654614ba2e2d3cd07edc8b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/3729978528244dd493610ba8a08a03f9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/de86b5bb9f8e46e29ce7a821165e4c1e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/809974d8936643fc9e59af0bd2108890, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/791fff8037254445b52a149e287af4fd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c8f1cd862f4f49c6b3ea5c1d612cf635, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6074539b8798424792973e489985a566, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8bbce9e860bb4a1da16be44a47185da2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c304cb59ac31464ca47d5ae6a8da18b0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0c33f9e08ba742c496c3b24b54d9f757, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/fadcaef66b8346c5a096cd972cb31de2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/89106903a3784bdb8c08e7ae290605f9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/efc120d5e1074deeb13afaf91d41ff4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/e8b5926cc2144a0d9bcd99b0801a54f4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/389bad7d4ec04f8189e233f563e2b5a0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/27e2f6bca969499a9e5fa9831720cba7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0fc9f674cdba4d3498cedd5b14a8398a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2ab33b8466eb4549a3dca561d9b3f0a6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c970ec487e44a68a6db793b5a5b046d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/148518bc08a541e38637d83189dc77f2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2d81f97527d2420a9aad67a52b661055] to archive 2024-12-08T01:14:34,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:14:34,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/746f05f7e3ac474f9ed75293dde7ac74 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/746f05f7e3ac474f9ed75293dde7ac74 2024-12-08T01:14:34,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a8d8769aaac1450bb6df6308919993df to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a8d8769aaac1450bb6df6308919993df 2024-12-08T01:14:34,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a1a9db5f9e494238a91fa3c0f4ddcbcc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a1a9db5f9e494238a91fa3c0f4ddcbcc 2024-12-08T01:14:34,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/d4c8f44ba1c24380ba4b09cee58d72ad to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/d4c8f44ba1c24380ba4b09cee58d72ad 2024-12-08T01:14:34,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/518663db9e144324b1738651337f481e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/518663db9e144324b1738651337f481e 2024-12-08T01:14:34,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2164eaf1d4344f7cb2f07a801f084303 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2164eaf1d4344f7cb2f07a801f084303 2024-12-08T01:14:34,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/f5fb77854ee04c71a1f8589542ba210a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/f5fb77854ee04c71a1f8589542ba210a 2024-12-08T01:14:34,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/4a8305da48154ad4b0356f366137a396 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/4a8305da48154ad4b0356f366137a396 2024-12-08T01:14:34,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/47a0e2f908c44de8bae74a0691b5b547 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/47a0e2f908c44de8bae74a0691b5b547 2024-12-08T01:14:34,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/7dbd973bd15448049944e96cf7cef912 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/7dbd973bd15448049944e96cf7cef912 2024-12-08T01:14:34,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a35eecb111b449259d2c2da1594d2f7c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/a35eecb111b449259d2c2da1594d2f7c 2024-12-08T01:14:34,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/08a178f1f7e541a3bda751bc883507bd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/08a178f1f7e541a3bda751bc883507bd 2024-12-08T01:14:34,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/785469833ad245bfa84557d18351e4f1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/785469833ad245bfa84557d18351e4f1 2024-12-08T01:14:34,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/1fed5dfc0d9e4474be3ae34b949a159b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/1fed5dfc0d9e4474be3ae34b949a159b 2024-12-08T01:14:34,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c94f364efc654614ba2e2d3cd07edc8b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c94f364efc654614ba2e2d3cd07edc8b 2024-12-08T01:14:34,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/3729978528244dd493610ba8a08a03f9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/3729978528244dd493610ba8a08a03f9 2024-12-08T01:14:34,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/de86b5bb9f8e46e29ce7a821165e4c1e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/de86b5bb9f8e46e29ce7a821165e4c1e 2024-12-08T01:14:34,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/809974d8936643fc9e59af0bd2108890 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/809974d8936643fc9e59af0bd2108890 2024-12-08T01:14:34,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/791fff8037254445b52a149e287af4fd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/791fff8037254445b52a149e287af4fd 2024-12-08T01:14:34,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c8f1cd862f4f49c6b3ea5c1d612cf635 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c8f1cd862f4f49c6b3ea5c1d612cf635 2024-12-08T01:14:34,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6074539b8798424792973e489985a566 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6074539b8798424792973e489985a566 2024-12-08T01:14:34,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8bbce9e860bb4a1da16be44a47185da2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8bbce9e860bb4a1da16be44a47185da2 2024-12-08T01:14:34,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c304cb59ac31464ca47d5ae6a8da18b0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/c304cb59ac31464ca47d5ae6a8da18b0 2024-12-08T01:14:34,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0c33f9e08ba742c496c3b24b54d9f757 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0c33f9e08ba742c496c3b24b54d9f757 2024-12-08T01:14:34,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/fadcaef66b8346c5a096cd972cb31de2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/fadcaef66b8346c5a096cd972cb31de2 2024-12-08T01:14:34,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/89106903a3784bdb8c08e7ae290605f9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/89106903a3784bdb8c08e7ae290605f9 2024-12-08T01:14:34,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/efc120d5e1074deeb13afaf91d41ff4e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/efc120d5e1074deeb13afaf91d41ff4e 2024-12-08T01:14:34,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/e8b5926cc2144a0d9bcd99b0801a54f4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/e8b5926cc2144a0d9bcd99b0801a54f4 2024-12-08T01:14:34,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/389bad7d4ec04f8189e233f563e2b5a0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/389bad7d4ec04f8189e233f563e2b5a0 2024-12-08T01:14:34,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/27e2f6bca969499a9e5fa9831720cba7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/27e2f6bca969499a9e5fa9831720cba7 2024-12-08T01:14:34,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0fc9f674cdba4d3498cedd5b14a8398a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/0fc9f674cdba4d3498cedd5b14a8398a 2024-12-08T01:14:34,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2ab33b8466eb4549a3dca561d9b3f0a6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2ab33b8466eb4549a3dca561d9b3f0a6 2024-12-08T01:14:34,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c970ec487e44a68a6db793b5a5b046d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c970ec487e44a68a6db793b5a5b046d 2024-12-08T01:14:34,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/148518bc08a541e38637d83189dc77f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/148518bc08a541e38637d83189dc77f2 2024-12-08T01:14:34,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2d81f97527d2420a9aad67a52b661055 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/2d81f97527d2420a9aad67a52b661055 2024-12-08T01:14:34,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/72a628dd799c4ff6931aff16f3f7c31e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/656d3c687609497a9e86084fb7e631bd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/91c131c7a3ac4417a1f7d15957f6b34a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/f75a77ad298b43bca9585a0c9823ecc6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0babee2f88cd4eabaf4653ab4340098e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/cb90ad3890fa4392998069fc96a59c1a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5472b08cd7fa4b18be15c63d8fdc6c43, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/93a809c19d6c4abfbee2e0b182136836, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/dcb4df59c3c2442790b5af02ce000ceb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/05e6288a26fa4d3a901e357f7049d653, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c2493ef71f440658ecbe895c1845f8b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/692f7d19f12d43bf99f2dd747f3cedf8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/2e643b294d634b42a0cbadcd838587b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4aacf05b158d4cc6aace4059f1adde6e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/350d281d50c34dcea88f3b08afcf6e30, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/8f143455530d40229fc04cf193235aa5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/c1fac3f7f8874850ba8a8e9883e01e95, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5dea50cbee594b368fb4f6cab6ae4b6b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c8c101a5d5c422286810775582fccf0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7450150cd68d41f7ac8f3589a92aa8f8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/fa3a0c75ef014001ab17706f0481ead1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/585c8b9a942948c38d1ac3bcda71549e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/1413c017144a4f0698be311556b6c5b3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7d0795422ce346d38bf0b0809f9e55f9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/b348719eebb34fb8b560091fd1437edf, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7f8de72f9a364906a4b19ef42bafbfec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4d713fc9edfc4c02b9e70071a9c93d97, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7cbf0cfc025a45afac0dfbc04e22903f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/021ad5e4d90a4d57a7e1a423ca2c0733, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/49c22fd1124749c9b5791b8ff6ba5b4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/156d9fc5278b4af1bfe36ca3f7c778a9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/be39c3ba70604481a9ef74eca547d277, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/3cc59dad62344e45a9080c427e49b4b1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/925fb79ad0ae469f8399a372d1a633cd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0b948ea9e01b4185894d8892f62d6c18] to archive 2024-12-08T01:14:34,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:14:34,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/72a628dd799c4ff6931aff16f3f7c31e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/72a628dd799c4ff6931aff16f3f7c31e 2024-12-08T01:14:34,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/656d3c687609497a9e86084fb7e631bd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/656d3c687609497a9e86084fb7e631bd 2024-12-08T01:14:34,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/91c131c7a3ac4417a1f7d15957f6b34a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/91c131c7a3ac4417a1f7d15957f6b34a 2024-12-08T01:14:34,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/f75a77ad298b43bca9585a0c9823ecc6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/f75a77ad298b43bca9585a0c9823ecc6 2024-12-08T01:14:34,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0babee2f88cd4eabaf4653ab4340098e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0babee2f88cd4eabaf4653ab4340098e 2024-12-08T01:14:34,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/cb90ad3890fa4392998069fc96a59c1a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/cb90ad3890fa4392998069fc96a59c1a 2024-12-08T01:14:34,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5472b08cd7fa4b18be15c63d8fdc6c43 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5472b08cd7fa4b18be15c63d8fdc6c43 2024-12-08T01:14:34,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/93a809c19d6c4abfbee2e0b182136836 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/93a809c19d6c4abfbee2e0b182136836 2024-12-08T01:14:34,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/dcb4df59c3c2442790b5af02ce000ceb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/dcb4df59c3c2442790b5af02ce000ceb 2024-12-08T01:14:34,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/05e6288a26fa4d3a901e357f7049d653 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/05e6288a26fa4d3a901e357f7049d653 2024-12-08T01:14:34,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c2493ef71f440658ecbe895c1845f8b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c2493ef71f440658ecbe895c1845f8b 2024-12-08T01:14:34,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/692f7d19f12d43bf99f2dd747f3cedf8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/692f7d19f12d43bf99f2dd747f3cedf8 2024-12-08T01:14:34,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/2e643b294d634b42a0cbadcd838587b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/2e643b294d634b42a0cbadcd838587b5 2024-12-08T01:14:34,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4aacf05b158d4cc6aace4059f1adde6e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4aacf05b158d4cc6aace4059f1adde6e 2024-12-08T01:14:34,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/350d281d50c34dcea88f3b08afcf6e30 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/350d281d50c34dcea88f3b08afcf6e30 2024-12-08T01:14:34,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/8f143455530d40229fc04cf193235aa5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/8f143455530d40229fc04cf193235aa5 2024-12-08T01:14:34,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/c1fac3f7f8874850ba8a8e9883e01e95 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/c1fac3f7f8874850ba8a8e9883e01e95 2024-12-08T01:14:34,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5dea50cbee594b368fb4f6cab6ae4b6b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5dea50cbee594b368fb4f6cab6ae4b6b 2024-12-08T01:14:34,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c8c101a5d5c422286810775582fccf0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/5c8c101a5d5c422286810775582fccf0 2024-12-08T01:14:34,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7450150cd68d41f7ac8f3589a92aa8f8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7450150cd68d41f7ac8f3589a92aa8f8 2024-12-08T01:14:34,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/fa3a0c75ef014001ab17706f0481ead1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/fa3a0c75ef014001ab17706f0481ead1 2024-12-08T01:14:34,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/585c8b9a942948c38d1ac3bcda71549e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/585c8b9a942948c38d1ac3bcda71549e 2024-12-08T01:14:34,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/1413c017144a4f0698be311556b6c5b3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/1413c017144a4f0698be311556b6c5b3 2024-12-08T01:14:34,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7d0795422ce346d38bf0b0809f9e55f9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7d0795422ce346d38bf0b0809f9e55f9 2024-12-08T01:14:34,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/b348719eebb34fb8b560091fd1437edf to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/b348719eebb34fb8b560091fd1437edf 2024-12-08T01:14:34,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7f8de72f9a364906a4b19ef42bafbfec to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7f8de72f9a364906a4b19ef42bafbfec 2024-12-08T01:14:34,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4d713fc9edfc4c02b9e70071a9c93d97 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/4d713fc9edfc4c02b9e70071a9c93d97 2024-12-08T01:14:34,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7cbf0cfc025a45afac0dfbc04e22903f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/7cbf0cfc025a45afac0dfbc04e22903f 2024-12-08T01:14:34,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/021ad5e4d90a4d57a7e1a423ca2c0733 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/021ad5e4d90a4d57a7e1a423ca2c0733 2024-12-08T01:14:34,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/49c22fd1124749c9b5791b8ff6ba5b4e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/49c22fd1124749c9b5791b8ff6ba5b4e 2024-12-08T01:14:34,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/156d9fc5278b4af1bfe36ca3f7c778a9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/156d9fc5278b4af1bfe36ca3f7c778a9 2024-12-08T01:14:34,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/be39c3ba70604481a9ef74eca547d277 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/be39c3ba70604481a9ef74eca547d277 2024-12-08T01:14:34,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/3cc59dad62344e45a9080c427e49b4b1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/3cc59dad62344e45a9080c427e49b4b1 2024-12-08T01:14:34,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/925fb79ad0ae469f8399a372d1a633cd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/925fb79ad0ae469f8399a372d1a633cd 2024-12-08T01:14:34,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0b948ea9e01b4185894d8892f62d6c18 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/0b948ea9e01b4185894d8892f62d6c18 2024-12-08T01:14:34,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6f32a4cdf941409390e32693350c3be2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3c9408d572ee41279974933750791fdd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6506dfaab68d4ade80895a9d07e9bdd0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0cfc5543e02b470c8abf22c3810942e7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3419f2b535334576b7f72e39ce5d5a58, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fbccc9d429474b7f8a9613dde6aca519, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/f6e466e6f4a94225be8a0ed80bb74622, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3d2e6256b9ad400db4a75a02b5357eed, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/5a858d0f6bdf47eebc151e029aac04be, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/519c3cbf362744aab6fe2d0406370d91, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/42f6e0bba28241e983ef1a97d14b2d2e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/e9201be6105f46fe909fc41b7ddebeee, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0d54a722743f48dc831778120ef5358f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/90b435b810ba40bd81461e067c947a30, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/c0129325fac443349644adef48abae71, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/732e2390bb72452bb05702fd367d7421, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/be51d3f57ed74f30b1e04ae5b17eac07, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b34b32f0e34545dd961f51d0d0b08978, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3b81e83a280e4d2e9278098cac928410, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/8804c50b361b42cb958e46135d01f4ef, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/1909c711be724f3ab985e9a4caebb93e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b97ef089ab6a45d1853384ba5d0bfda5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/4c96dd09e60f4845bad37ec253e7241f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fa6ad2f57a744011bac9f8543241cb77, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/a5c426e638d04ec88681eab4fde07443, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/7cdc7c32d7804dd3a79d51b257711f5b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/92eb195ebf71460bb50f657da601dddc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/62b2958593fc40e9aabca69a337251f3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/28d9bcb13e1c49eca39b8bac89854571, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/2523299fce9648ee8353cbaacf2050e7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/41959547d7924c949c7170e84b4785a0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/d6d3ee97e464492e998fea422d085d5f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b26fe07b1f3c4628b0d3de45737c5f48, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/526f3df993334cd78d59aa266799d85c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/de8d153cfcad422db468d25915f9fafb] to archive 2024-12-08T01:14:34,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:14:34,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6f32a4cdf941409390e32693350c3be2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6f32a4cdf941409390e32693350c3be2 2024-12-08T01:14:34,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3c9408d572ee41279974933750791fdd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3c9408d572ee41279974933750791fdd 2024-12-08T01:14:34,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6506dfaab68d4ade80895a9d07e9bdd0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/6506dfaab68d4ade80895a9d07e9bdd0 2024-12-08T01:14:34,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0cfc5543e02b470c8abf22c3810942e7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0cfc5543e02b470c8abf22c3810942e7 2024-12-08T01:14:34,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3419f2b535334576b7f72e39ce5d5a58 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3419f2b535334576b7f72e39ce5d5a58 2024-12-08T01:14:34,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fbccc9d429474b7f8a9613dde6aca519 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fbccc9d429474b7f8a9613dde6aca519 2024-12-08T01:14:34,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/f6e466e6f4a94225be8a0ed80bb74622 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/f6e466e6f4a94225be8a0ed80bb74622 2024-12-08T01:14:34,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3d2e6256b9ad400db4a75a02b5357eed to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3d2e6256b9ad400db4a75a02b5357eed 2024-12-08T01:14:34,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/5a858d0f6bdf47eebc151e029aac04be to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/5a858d0f6bdf47eebc151e029aac04be 2024-12-08T01:14:34,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/519c3cbf362744aab6fe2d0406370d91 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/519c3cbf362744aab6fe2d0406370d91 2024-12-08T01:14:34,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/42f6e0bba28241e983ef1a97d14b2d2e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/42f6e0bba28241e983ef1a97d14b2d2e 2024-12-08T01:14:34,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/e9201be6105f46fe909fc41b7ddebeee to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/e9201be6105f46fe909fc41b7ddebeee 2024-12-08T01:14:34,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0d54a722743f48dc831778120ef5358f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/0d54a722743f48dc831778120ef5358f 2024-12-08T01:14:34,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/90b435b810ba40bd81461e067c947a30 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/90b435b810ba40bd81461e067c947a30 2024-12-08T01:14:34,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/c0129325fac443349644adef48abae71 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/c0129325fac443349644adef48abae71 2024-12-08T01:14:34,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/732e2390bb72452bb05702fd367d7421 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/732e2390bb72452bb05702fd367d7421 2024-12-08T01:14:34,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/be51d3f57ed74f30b1e04ae5b17eac07 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/be51d3f57ed74f30b1e04ae5b17eac07 2024-12-08T01:14:34,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b34b32f0e34545dd961f51d0d0b08978 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b34b32f0e34545dd961f51d0d0b08978 2024-12-08T01:14:34,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3b81e83a280e4d2e9278098cac928410 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/3b81e83a280e4d2e9278098cac928410 2024-12-08T01:14:34,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/8804c50b361b42cb958e46135d01f4ef to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/8804c50b361b42cb958e46135d01f4ef 2024-12-08T01:14:34,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/1909c711be724f3ab985e9a4caebb93e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/1909c711be724f3ab985e9a4caebb93e 2024-12-08T01:14:34,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b97ef089ab6a45d1853384ba5d0bfda5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b97ef089ab6a45d1853384ba5d0bfda5 2024-12-08T01:14:34,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/4c96dd09e60f4845bad37ec253e7241f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/4c96dd09e60f4845bad37ec253e7241f 2024-12-08T01:14:34,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fa6ad2f57a744011bac9f8543241cb77 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/fa6ad2f57a744011bac9f8543241cb77 2024-12-08T01:14:34,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/a5c426e638d04ec88681eab4fde07443 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/a5c426e638d04ec88681eab4fde07443 2024-12-08T01:14:34,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/7cdc7c32d7804dd3a79d51b257711f5b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/7cdc7c32d7804dd3a79d51b257711f5b 2024-12-08T01:14:34,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/92eb195ebf71460bb50f657da601dddc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/92eb195ebf71460bb50f657da601dddc 2024-12-08T01:14:34,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/62b2958593fc40e9aabca69a337251f3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/62b2958593fc40e9aabca69a337251f3 2024-12-08T01:14:34,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/28d9bcb13e1c49eca39b8bac89854571 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/28d9bcb13e1c49eca39b8bac89854571 2024-12-08T01:14:34,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/2523299fce9648ee8353cbaacf2050e7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/2523299fce9648ee8353cbaacf2050e7 2024-12-08T01:14:34,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/41959547d7924c949c7170e84b4785a0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/41959547d7924c949c7170e84b4785a0 2024-12-08T01:14:34,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/d6d3ee97e464492e998fea422d085d5f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/d6d3ee97e464492e998fea422d085d5f 2024-12-08T01:14:34,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b26fe07b1f3c4628b0d3de45737c5f48 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/b26fe07b1f3c4628b0d3de45737c5f48 2024-12-08T01:14:34,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/526f3df993334cd78d59aa266799d85c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/526f3df993334cd78d59aa266799d85c 2024-12-08T01:14:34,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/de8d153cfcad422db468d25915f9fafb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/de8d153cfcad422db468d25915f9fafb 2024-12-08T01:14:34,315 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/recovered.edits/575.seqid, newMaxSeqId=575, maxSeqId=1 2024-12-08T01:14:34,316 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. 2024-12-08T01:14:34,316 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 13ab931b3a97ca4195bbab8a55597446: Waiting for close lock at 1733620472892Running coprocessor pre-close hooks at 1733620472892Disabling compacts and flushes for region at 1733620472892Disabling writes for close at 1733620472892Obtaining lock to block concurrent updates at 1733620472893 (+1 ms)Preparing flush snapshotting stores in 13ab931b3a97ca4195bbab8a55597446 at 1733620472893Finished memstore snapshotting TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733620472894 (+1 ms)Flushing stores of TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446. at 1733620472895 (+1 ms)Flushing 13ab931b3a97ca4195bbab8a55597446/A: creating writer at 1733620472895Flushing 13ab931b3a97ca4195bbab8a55597446/A: appending metadata at 1733620472899 (+4 ms)Flushing 13ab931b3a97ca4195bbab8a55597446/A: closing flushed file at 1733620472899Flushing 13ab931b3a97ca4195bbab8a55597446/B: creating writer at 1733620473316 (+417 ms)Flushing 13ab931b3a97ca4195bbab8a55597446/B: appending metadata at 1733620473320 (+4 ms)Flushing 13ab931b3a97ca4195bbab8a55597446/B: closing flushed file at 1733620473320Flushing 13ab931b3a97ca4195bbab8a55597446/C: creating writer at 1733620473730 (+410 ms)Flushing 13ab931b3a97ca4195bbab8a55597446/C: appending metadata at 1733620473734 (+4 ms)Flushing 13ab931b3a97ca4195bbab8a55597446/C: closing flushed file at 1733620473734Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d81334: reopening flushed file at 1733620474150 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59c69cc9: reopening flushed file at 1733620474157 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62600e56: reopening flushed file at 1733620474163 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 13ab931b3a97ca4195bbab8a55597446 in 1278ms, sequenceid=572, compaction requested=true at 1733620474170 (+7 ms)Writing region close event to WAL at 1733620474312 (+142 ms)Running coprocessor post-close hooks at 1733620474316 (+4 ms)Closed at 1733620474316 2024-12-08T01:14:34,319 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:34,319 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=13ab931b3a97ca4195bbab8a55597446, regionState=CLOSED 2024-12-08T01:14:34,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:14:34,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-08T01:14:34,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 13ab931b3a97ca4195bbab8a55597446, server=96982154c400,44843,1733620436630 in 1.5900 sec 2024-12-08T01:14:34,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T01:14:34,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13ab931b3a97ca4195bbab8a55597446, UNASSIGN in 1.5960 sec 2024-12-08T01:14:34,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-08T01:14:34,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6020 sec 2024-12-08T01:14:34,328 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620474328"}]},"ts":"1733620474328"} 2024-12-08T01:14:34,330 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T01:14:34,330 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T01:14:34,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6170 sec 2024-12-08T01:14:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T01:14:34,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:34,870 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T01:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:34,876 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-08T01:14:34,878 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:34,883 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:34,887 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/recovered.edits] 2024-12-08T01:14:34,891 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/54ada86d855e4430aa8082968e781de7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/54ada86d855e4430aa8082968e781de7 2024-12-08T01:14:34,892 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/5a48a1bf61f94ccb8da96739a23cce78 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/5a48a1bf61f94ccb8da96739a23cce78 2024-12-08T01:14:34,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/64147abfb786445cbedaaaad40ce60fe to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/64147abfb786445cbedaaaad40ce60fe 2024-12-08T01:14:34,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c3ca7544c214a058df703f8320721ff to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/6c3ca7544c214a058df703f8320721ff 2024-12-08T01:14:34,897 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8ad0656c103d4f82a5840d6f33a8792f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/A/8ad0656c103d4f82a5840d6f33a8792f 2024-12-08T01:14:34,899 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/38b00f9f78384fdab8ee3b9f41efea3a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/38b00f9f78384fdab8ee3b9f41efea3a 2024-12-08T01:14:34,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/6706b369a16d4b099d219dd14a04b7e9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/6706b369a16d4b099d219dd14a04b7e9 2024-12-08T01:14:34,901 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/69fc0e1ff29745228913abe18d3a1eae to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/69fc0e1ff29745228913abe18d3a1eae 2024-12-08T01:14:34,902 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/d358f7313dd7485480541bca47464c9b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/d358f7313dd7485480541bca47464c9b 2024-12-08T01:14:34,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/e290a2c340c442d3a9aef080d2baf1d4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/B/e290a2c340c442d3a9aef080d2baf1d4 2024-12-08T01:14:34,906 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/15d1ccbbda1f41948ad8bdd8d0d78632 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/15d1ccbbda1f41948ad8bdd8d0d78632 2024-12-08T01:14:34,907 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/417c03c3bd4b47039950749374a1741a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/417c03c3bd4b47039950749374a1741a 2024-12-08T01:14:34,908 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/560a34d173ec410fa68451bdcb79544a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/560a34d173ec410fa68451bdcb79544a 2024-12-08T01:14:34,909 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/ad4128dae7434e9599fb3be3d21648e9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/ad4128dae7434e9599fb3be3d21648e9 2024-12-08T01:14:34,910 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/eff640b1eac044078b5b86c8719692e4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/C/eff640b1eac044078b5b86c8719692e4 2024-12-08T01:14:34,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/recovered.edits/575.seqid to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446/recovered.edits/575.seqid 2024-12-08T01:14:34,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/13ab931b3a97ca4195bbab8a55597446 2024-12-08T01:14:34,913 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T01:14:34,918 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:34,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-08T01:14:34,928 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T01:14:34,959 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T01:14:34,961 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:34,961 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T01:14:34,961 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733620474961"}]},"ts":"9223372036854775807"} 2024-12-08T01:14:34,966 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T01:14:34,966 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 13ab931b3a97ca4195bbab8a55597446, NAME => 'TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T01:14:34,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T01:14:34,967 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733620474966"}]},"ts":"9223372036854775807"} 2024-12-08T01:14:34,970 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T01:14:34,971 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:34,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-12-08T01:14:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-08T01:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T01:14:34,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:34,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/96982154c400:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/96982154c400:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;96982154c400:44843-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-207628044_22 at /127.0.0.1:36186 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9615 (was 10212) 2024-12-08T01:14:35,002 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=326, ProcessCount=11, AvailableMemoryMB=9615 2024-12-08T01:14:35,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:14:35,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:14:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:35,006 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:14:35,007 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:35,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-12-08T01:14:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T01:14:35,008 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:14:35,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741957_1133 (size=1061) 2024-12-08T01:14:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T01:14:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T01:14:35,420 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:14:35,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741958_1134 (size=53) 2024-12-08T01:14:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T01:14:35,870 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:35,870 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 56865e84e13eaff78d5439ba323ab5f2, disabling compactions & flushes 2024-12-08T01:14:35,871 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:35,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:35,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. after waiting 0 ms 2024-12-08T01:14:35,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:35,871 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:35,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56865e84e13eaff78d5439ba323ab5f2: Waiting for close lock at 1733620475870Disabling compacts and flushes for region at 1733620475870Disabling writes for close at 1733620475871 (+1 ms)Writing region close event to WAL at 1733620475871Closed at 1733620475871 2024-12-08T01:14:35,874 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:14:35,875 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733620475875"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620475875"}]},"ts":"1733620475875"} 2024-12-08T01:14:35,878 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:14:35,879 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:14:35,880 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620475879"}]},"ts":"1733620475879"} 2024-12-08T01:14:35,881 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T01:14:35,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, ASSIGN}] 2024-12-08T01:14:35,883 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, ASSIGN 2024-12-08T01:14:35,884 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, ASSIGN; state=OFFLINE, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=false 2024-12-08T01:14:36,035 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:36,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, ASSIGN because future has completed 2024-12-08T01:14:36,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630}] 2024-12-08T01:14:36,091 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T01:14:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T01:14:36,202 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:36,203 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:14:36,203 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,204 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:36,204 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,204 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,207 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,208 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:36,209 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56865e84e13eaff78d5439ba323ab5f2 columnFamilyName A 2024-12-08T01:14:36,209 DEBUG [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:36,210 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(327): Store=56865e84e13eaff78d5439ba323ab5f2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:36,210 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,211 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:36,211 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56865e84e13eaff78d5439ba323ab5f2 columnFamilyName B 2024-12-08T01:14:36,211 DEBUG [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:36,212 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(327): Store=56865e84e13eaff78d5439ba323ab5f2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:36,212 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,213 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:36,213 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56865e84e13eaff78d5439ba323ab5f2 columnFamilyName C 2024-12-08T01:14:36,213 DEBUG [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:36,214 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(327): Store=56865e84e13eaff78d5439ba323ab5f2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:36,214 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:36,214 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,215 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,215 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,216 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,216 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,217 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:14:36,218 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,220 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:14:36,221 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 56865e84e13eaff78d5439ba323ab5f2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68607090, jitterRate=0.0223253071308136}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:14:36,221 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 56865e84e13eaff78d5439ba323ab5f2: Running coprocessor pre-open hook at 1733620476205Writing region info on filesystem at 1733620476205Initializing all the Stores at 1733620476206 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620476206Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620476206Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620476206Cleaning up temporary data from old regions at 1733620476216 (+10 ms)Running coprocessor post-open hooks at 1733620476221 (+5 ms)Region opened successfully at 1733620476223 (+2 ms) 2024-12-08T01:14:36,224 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., pid=38, masterSystemTime=1733620476196 2024-12-08T01:14:36,227 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:36,227 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:36,227 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:36,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:14:36,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-08T01:14:36,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 in 188 msec 2024-12-08T01:14:36,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-08T01:14:36,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, ASSIGN in 350 msec 2024-12-08T01:14:36,235 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:14:36,235 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620476235"}]},"ts":"1733620476235"} 2024-12-08T01:14:36,236 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T01:14:36,237 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:14:36,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2330 sec 2024-12-08T01:14:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-08T01:14:37,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:37,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:37,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:37,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:37,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:37,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53186973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:37,164 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:37,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:37,168 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:14:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:14:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18303f2f 2024-12-08T01:14:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:14:37,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:14:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:14:37,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:14:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T01:14:37,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741959_1135 (size=1097) 2024-12-08T01:14:37,598 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T01:14:37,598 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T01:14:37,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:14:37,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, REOPEN/MOVE}] 2024-12-08T01:14:37,613 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, REOPEN/MOVE 2024-12-08T01:14:37,614 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:37,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, REOPEN/MOVE because future has completed 2024-12-08T01:14:37,616 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:14:37,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630}] 2024-12-08T01:14:37,769 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:37,769 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:14:37,770 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 56865e84e13eaff78d5439ba323ab5f2, disabling compactions & flushes 2024-12-08T01:14:37,770 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:37,770 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:37,770 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. after waiting 0 ms 2024-12-08T01:14:37,770 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:37,776 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T01:14:37,777 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:37,778 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 56865e84e13eaff78d5439ba323ab5f2: Waiting for close lock at 1733620477769Running coprocessor pre-close hooks at 1733620477769Disabling compacts and flushes for region at 1733620477769Disabling writes for close at 1733620477770 (+1 ms)Writing region close event to WAL at 1733620477770Running coprocessor post-close hooks at 1733620477777 (+7 ms)Closed at 1733620477777 2024-12-08T01:14:37,778 WARN [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 56865e84e13eaff78d5439ba323ab5f2 to self. 2024-12-08T01:14:37,781 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:37,782 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=CLOSED 2024-12-08T01:14:37,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:14:37,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-12-08T01:14:37,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 in 170 msec 2024-12-08T01:14:37,790 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, REOPEN/MOVE; state=CLOSED, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=true 2024-12-08T01:14:37,941 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:37,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, REOPEN/MOVE because future has completed 2024-12-08T01:14:37,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630}] 2024-12-08T01:14:38,098 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,098 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:14:38,099 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,099 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:14:38,099 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,099 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,101 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,102 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:38,107 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56865e84e13eaff78d5439ba323ab5f2 columnFamilyName A 2024-12-08T01:14:38,109 DEBUG [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:38,109 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(327): Store=56865e84e13eaff78d5439ba323ab5f2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:38,110 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,111 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:38,111 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56865e84e13eaff78d5439ba323ab5f2 columnFamilyName B 2024-12-08T01:14:38,111 DEBUG [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:38,112 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(327): Store=56865e84e13eaff78d5439ba323ab5f2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:38,112 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,112 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:14:38,113 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56865e84e13eaff78d5439ba323ab5f2 columnFamilyName C 2024-12-08T01:14:38,113 DEBUG [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:38,113 INFO [StoreOpener-56865e84e13eaff78d5439ba323ab5f2-1 {}] regionserver.HStore(327): Store=56865e84e13eaff78d5439ba323ab5f2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:14:38,113 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,113 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,114 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,115 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,116 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,116 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,116 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:14:38,117 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,118 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 56865e84e13eaff78d5439ba323ab5f2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59080795, jitterRate=-0.11962755024433136}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:14:38,118 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,120 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 56865e84e13eaff78d5439ba323ab5f2: Running coprocessor pre-open hook at 1733620478099Writing region info on filesystem at 1733620478099Initializing all the Stores at 1733620478100 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620478100Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620478100Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620478100Cleaning up temporary data from old regions at 1733620478116 (+16 ms)Running coprocessor post-open hooks at 1733620478118 (+2 ms)Region opened successfully at 1733620478120 (+2 ms) 2024-12-08T01:14:38,121 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., pid=43, masterSystemTime=1733620478095 2024-12-08T01:14:38,122 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,122 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,123 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=OPEN, openSeqNum=5, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:14:38,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:14:38,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-12-08T01:14:38,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 in 182 msec 2024-12-08T01:14:38,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-08T01:14:38,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, REOPEN/MOVE in 515 msec 2024-12-08T01:14:38,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-08T01:14:38,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-12-08T01:14:38,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 953 msec 2024-12-08T01:14:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-08T01:14:38,136 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:38,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607d88b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,140 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb2646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,142 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237a3912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,143 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32836003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5321e01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,147 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee33ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,149 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4608c021, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@601c9631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,152 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759103da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,153 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f58c6d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,155 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e80640f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,156 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd47ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,158 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,160 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c22aa02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,161 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6874eb42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,163 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47775e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:14:38,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:14:38,164 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:14:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:14:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:14:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df4496f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:14:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:14:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:38,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:14:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1713fe00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:14:38,167 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,167 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,167 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,167 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:38,168 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,168 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,169 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-08T01:14:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,170 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,171 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:14:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,171 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,172 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:14:38,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T01:14:38,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:14:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:14:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:38,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120850d5ad6eb0c7462e88bfcf7457e7f55b_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620478187/Put/seqid=0 2024-12-08T01:14:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620478218, max timestamp=1733620478246Num uniques 859; 2024-12-08T01:14:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620478218, max timestamp=1733620478246Num uniques 859; 2024-12-08T01:14:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:38,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620478218, max timestamp=1733620478246Num uniques 859; 2024-12-08T01:14:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:38,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741960_1136 (size=17034) 2024-12-08T01:14:38,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:38,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120850d5ad6eb0c7462e88bfcf7457e7f55b_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120850d5ad6eb0c7462e88bfcf7457e7f55b_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e6d4f8abb2641c788ce1c62fbb17d1d, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T01:14:38,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e6d4f8abb2641c788ce1c62fbb17d1d is 175, key is test_row_0/A:col10/1733620478187/Put/seqid=0 2024-12-08T01:14:38,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620538277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620538279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620538279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620538289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620538289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741961_1137 (size=48139) 2024-12-08T01:14:38,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e6d4f8abb2641c788ce1c62fbb17d1d 2024-12-08T01:14:38,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/1f0b4c5ea7f74dceba6b605d07d18491 is 50, key is test_row_0/B:col10/1733620478187/Put/seqid=0 2024-12-08T01:14:38,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741962_1138 (size=12001) 2024-12-08T01:14:38,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/1f0b4c5ea7f74dceba6b605d07d18491 2024-12-08T01:14:38,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/47b01471ff7943ba95ed3d83f1990c43 is 50, key is test_row_0/C:col10/1733620478187/Put/seqid=0 2024-12-08T01:14:38,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620538402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620538403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620538403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620538403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620538404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741963_1139 (size=12001) 2024-12-08T01:14:38,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:38,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:38,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T01:14:38,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620538612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620538612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620538612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620538613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620538613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:38,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:38,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:38,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:38,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T01:14:38,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/47b01471ff7943ba95ed3d83f1990c43 2024-12-08T01:14:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e6d4f8abb2641c788ce1c62fbb17d1d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d 2024-12-08T01:14:38,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d, entries=250, sequenceid=14, filesize=47.0 K 2024-12-08T01:14:38,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/1f0b4c5ea7f74dceba6b605d07d18491 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/1f0b4c5ea7f74dceba6b605d07d18491 2024-12-08T01:14:38,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/1f0b4c5ea7f74dceba6b605d07d18491, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T01:14:38,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/47b01471ff7943ba95ed3d83f1990c43 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/47b01471ff7943ba95ed3d83f1990c43 2024-12-08T01:14:38,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/47b01471ff7943ba95ed3d83f1990c43, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T01:14:38,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 56865e84e13eaff78d5439ba323ab5f2 in 672ms, sequenceid=14, compaction requested=false 2024-12-08T01:14:38,861 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T01:14:38,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-08T01:14:38,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620478276, max timestamp=1733620478925Num uniques 1118; 2024-12-08T01:14:38,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-08T01:14:38,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620478276, max timestamp=1733620478925Num uniques 1118; 2024-12-08T01:14:38,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620478276, max timestamp=1733620478925Num uniques 1118; 2024-12-08T01:14:38,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-08T01:14:38,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-08T01:14:38,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:38,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:38,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:38,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:38,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:38,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:38,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:38,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a30a77e22fd341ada8d561f846ba2c43_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620478925/Put/seqid=0 2024-12-08T01:14:38,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741964_1140 (size=14594) 2024-12-08T01:14:38,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:38,975 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a30a77e22fd341ada8d561f846ba2c43_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a30a77e22fd341ada8d561f846ba2c43_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:38,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620478971, max timestamp=1733620478974Num uniques 859; 2024-12-08T01:14:38,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:38,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620478971, max timestamp=1733620478974Num uniques 859; 2024-12-08T01:14:38,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:38,979 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/4abfcae108b74b0bb4eefcde6d0b914e, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:38,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/4abfcae108b74b0bb4eefcde6d0b914e is 175, key is test_row_0/A:col10/1733620478925/Put/seqid=0 2024-12-08T01:14:38,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620478971, max timestamp=1733620478974Num uniques 859; 2024-12-08T01:14:38,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:38,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741965_1141 (size=39549) 2024-12-08T01:14:38,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/4abfcae108b74b0bb4eefcde6d0b914e 2024-12-08T01:14:39,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/62fdafcd37774e4b89b65b295994b4f8 is 50, key is test_row_0/B:col10/1733620478925/Put/seqid=0 2024-12-08T01:14:39,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620479003, max timestamp=1733620479013Num uniques 1018; 2024-12-08T01:14:39,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T01:14:39,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9970, totalHeapSize=22912, min timestamp=1733620479003, max timestamp=1733620479013Num uniques 1018; 2024-12-08T01:14:39,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=9970, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T01:14:39,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733620479003, max timestamp=1733620479013Num uniques 1018; 2024-12-08T01:14:39,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46611, new segment data size=7185, suffix heap size=133840,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=157 2024-12-08T01:14:39,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741966_1142 (size=12001) 2024-12-08T01:14:39,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/62fdafcd37774e4b89b65b295994b4f8 2024-12-08T01:14:39,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/093ea805583a4ea0b741a8af6c808eac is 50, key is test_row_0/C:col10/1733620478925/Put/seqid=0 2024-12-08T01:14:39,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733620479027, max timestamp=1733620479049Num uniques 1077; 2024-12-08T01:14:39,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-08T01:14:39,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733620479027, max timestamp=1733620479049Num uniques 1077; 2024-12-08T01:14:39,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-08T01:14:39,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733620479037, max timestamp=1733620479050Num uniques 1016; 2024-12-08T01:14:39,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46536, new segment data size=8102, suffix heap size=133632,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=177 2024-12-08T01:14:39,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741967_1143 (size=12001) 2024-12-08T01:14:39,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733620479075, max timestamp=1733620479089Num uniques 1186; 2024-12-08T01:14:39,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-08T01:14:39,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733620479075, max timestamp=1733620479090Num uniques 1036; 2024-12-08T01:14:39,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733620479075, max timestamp=1733620479090Num uniques 1186; 2024-12-08T01:14:39,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47442, new segment data size=7050, suffix heap size=135712,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=154 2024-12-08T01:14:39,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54306, new segment data size=8506, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-08T01:14:39,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:39,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:39,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733620479113, max timestamp=1733620479127Num uniques 1045; 2024-12-08T01:14:39,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733620479113, max timestamp=1733620479127Num uniques 1045; 2024-12-08T01:14:39,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-08T01:14:39,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-08T01:14:39,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733620479119, max timestamp=1733620479128Num uniques 1013; 2024-12-08T01:14:39,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46398, new segment data size=8930, suffix heap size=133320,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=195 2024-12-08T01:14:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620539158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13914, totalHeapSize=31856, min timestamp=1733620479152, max timestamp=1733620479163Num uniques 1154; 2024-12-08T01:14:39,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=13914, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-08T01:14:39,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13914, totalHeapSize=31856, min timestamp=1733620479152, max timestamp=1733620479163Num uniques 1054; 2024-12-08T01:14:39,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48264, new segment data size=13914, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-12-08T01:14:39,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13914, totalHeapSize=31856, min timestamp=1733620479152, max timestamp=1733620479163Num uniques 1154; 2024-12-08T01:14:39,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=13914, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-08T01:14:39,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620539200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733620479191, max timestamp=1733620479204Num uniques 1163; 2024-12-08T01:14:39,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733620479191, max timestamp=1733620479204Num uniques 1163; 2024-12-08T01:14:39,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-08T01:14:39,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-08T01:14:39,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12035, totalHeapSize=27592, min timestamp=1733620479191, max timestamp=1733620479204Num uniques 1163; 2024-12-08T01:14:39,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53255, new segment data size=12035, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-08T01:14:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620539246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733620479234, max timestamp=1733620479249Num uniques 1122; 2024-12-08T01:14:39,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-08T01:14:39,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733620479234, max timestamp=1733620479249Num uniques 1122; 2024-12-08T01:14:39,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-08T01:14:39,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733620479234, max timestamp=1733620479249Num uniques 1122; 2024-12-08T01:14:39,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-08T01:14:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=381, cellSize=17446, totalHeapSize=39864, min timestamp=1733620479272, max timestamp=1733620479296Num uniques 1031; 2024-12-08T01:14:39,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=17446, suffix heap size=135192,new segment heap size=39864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=381 2024-12-08T01:14:39,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=381, cellSize=17446, totalHeapSize=39864, min timestamp=1733620479272, max timestamp=1733620479296Num uniques 1031; 2024-12-08T01:14:39,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=17446, suffix heap size=135192,new segment heap size=39864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=381 2024-12-08T01:14:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=381, cellSize=17446, totalHeapSize=39864, min timestamp=1733620479272, max timestamp=1733620479296Num uniques 1031; 2024-12-08T01:14:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=17446, suffix heap size=135192,new segment heap size=39864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=381 2024-12-08T01:14:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T01:14:39,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=390, cellSize=17860, totalHeapSize=40800, min timestamp=1733620479310, max timestamp=1733620479339Num uniques 1240; 2024-12-08T01:14:39,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56790, new segment data size=17860, suffix heap size=156928,new segment heap size=40800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1240, new segment cells count=390 2024-12-08T01:14:39,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=390, cellSize=17860, totalHeapSize=40800, min timestamp=1733620479310, max timestamp=1733620479339Num uniques 1240; 2024-12-08T01:14:39,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=56790, new segment data size=17860, suffix heap size=156928,new segment heap size=40800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1240, new segment cells count=390 2024-12-08T01:14:39,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=390, cellSize=17860, totalHeapSize=40800, min timestamp=1733620479310, max timestamp=1733620479339Num uniques 1240; 2024-12-08T01:14:39,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=56790, new segment data size=17860, suffix heap size=156928,new segment heap size=40800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1240, new segment cells count=390 2024-12-08T01:14:39,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620539371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620539372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620539374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620539375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620539376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:39,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:39,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/093ea805583a4ea0b741a8af6c808eac 2024-12-08T01:14:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620539482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620539482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/4abfcae108b74b0bb4eefcde6d0b914e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e 2024-12-08T01:14:39,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e, entries=200, sequenceid=52, filesize=38.6 K 2024-12-08T01:14:39,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/62fdafcd37774e4b89b65b295994b4f8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/62fdafcd37774e4b89b65b295994b4f8 2024-12-08T01:14:39,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620539492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620539493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620539493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/62fdafcd37774e4b89b65b295994b4f8, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T01:14:39,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/093ea805583a4ea0b741a8af6c808eac as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/093ea805583a4ea0b741a8af6c808eac 2024-12-08T01:14:39,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/093ea805583a4ea0b741a8af6c808eac, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T01:14:39,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=161.02 KB/164880 for 56865e84e13eaff78d5439ba323ab5f2 in 589ms, sequenceid=52, compaction requested=false 2024-12-08T01:14:39,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:39,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T01:14:39,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,560 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=161.02 KB heapSize=443.02 KB 2024-12-08T01:14:39,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:39,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe70492f2f7347b886132d2541b23c1a_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620479374/Put/seqid=0 2024-12-08T01:14:39,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741968_1144 (size=12304) 2024-12-08T01:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,643 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe70492f2f7347b886132d2541b23c1a_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe70492f2f7347b886132d2541b23c1a_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/8ac9be93943c442b9b17fe69eb75e4b5, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:39,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/8ac9be93943c442b9b17fe69eb75e4b5 is 175, key is test_row_0/A:col10/1733620479374/Put/seqid=0 2024-12-08T01:14:39,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741969_1145 (size=31105) 2024-12-08T01:14:39,655 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/8ac9be93943c442b9b17fe69eb75e4b5 2024-12-08T01:14:39,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/3afe240033bf4b00bbab5e7d338be9c1 is 50, key is test_row_0/B:col10/1733620479374/Put/seqid=0 2024-12-08T01:14:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741970_1146 (size=12151) 2024-12-08T01:14:39,682 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/3afe240033bf4b00bbab5e7d338be9c1 2024-12-08T01:14:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/f72e9910f03e4be2a1ae113584393fa2 is 50, key is test_row_0/C:col10/1733620479374/Put/seqid=0 2024-12-08T01:14:39,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:39,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620539702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620539702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620539704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620539706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620539710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741971_1147 (size=12151) 2024-12-08T01:14:39,725 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/f72e9910f03e4be2a1ae113584393fa2 2024-12-08T01:14:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/8ac9be93943c442b9b17fe69eb75e4b5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5 2024-12-08T01:14:39,740 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5, entries=150, sequenceid=243, filesize=30.4 K 2024-12-08T01:14:39,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/3afe240033bf4b00bbab5e7d338be9c1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3afe240033bf4b00bbab5e7d338be9c1 2024-12-08T01:14:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,748 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3afe240033bf4b00bbab5e7d338be9c1, entries=150, sequenceid=243, filesize=11.9 K 2024-12-08T01:14:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/f72e9910f03e4be2a1ae113584393fa2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/f72e9910f03e4be2a1ae113584393fa2 2024-12-08T01:14:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,757 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/f72e9910f03e4be2a1ae113584393fa2, entries=150, sequenceid=243, filesize=11.9 K 2024-12-08T01:14:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,759 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~442.97 KB/453600, currentSize=26.84 KB/27480 for 56865e84e13eaff78d5439ba323ab5f2 in 199ms, sequenceid=243, compaction requested=true 2024-12-08T01:14:39,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:39,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:39,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-08T01:14:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-08T01:14:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-08T01:14:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-12-08T01:14:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5980 sec 2024-12-08T01:14:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:39,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e92b5e58716540ec800ee2ba23d0ec66_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:39,851 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:14:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741972_1148 (size=17284) 2024-12-08T01:14:39,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:39,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e92b5e58716540ec800ee2ba23d0ec66_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e92b5e58716540ec800ee2ba23d0ec66_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:39,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/15d5d406283e4f48a1b2512da65c6b76, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:39,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/15d5d406283e4f48a1b2512da65c6b76 is 175, key is test_row_0/A:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:39,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620479902, max timestamp=1733620479915Num uniques 859; 2024-12-08T01:14:39,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620479902, max timestamp=1733620479915Num uniques 859; 2024-12-08T01:14:39,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:39,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:39,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620479902, max timestamp=1733620479915Num uniques 859; 2024-12-08T01:14:39,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:39,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741973_1149 (size=48385) 2024-12-08T01:14:39,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620539969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620539971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620540013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620540013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620540022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620540082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620540082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620540302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620540302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T01:14:40,322 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:40,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-08T01:14:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:40,325 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:40,326 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:40,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:40,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/15d5d406283e4f48a1b2512da65c6b76 2024-12-08T01:14:40,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/632ac03afd7e42e7af2a3f71979a37c6 is 50, key is test_row_0/B:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:40,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741974_1150 (size=9757) 2024-12-08T01:14:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:40,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T01:14:40,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620540522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620540522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620540544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620540612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620540612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:40,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T01:14:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:40,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/632ac03afd7e42e7af2a3f71979a37c6 2024-12-08T01:14:40,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/3e114f4167344f0f9539bf434a736caf is 50, key is test_row_0/C:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:40,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741975_1151 (size=9757) 2024-12-08T01:14:40,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T01:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:40,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T01:14:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:40,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:41,087 INFO [master/96982154c400:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T01:14:41,087 INFO [master/96982154c400:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T01:14:41,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T01:14:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:41,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620541152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620541152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/3e114f4167344f0f9539bf434a736caf 2024-12-08T01:14:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/15d5d406283e4f48a1b2512da65c6b76 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76 2024-12-08T01:14:41,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76, entries=250, sequenceid=253, filesize=47.3 K 2024-12-08T01:14:41,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/632ac03afd7e42e7af2a3f71979a37c6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/632ac03afd7e42e7af2a3f71979a37c6 2024-12-08T01:14:41,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/632ac03afd7e42e7af2a3f71979a37c6, entries=100, sequenceid=253, filesize=9.5 K 2024-12-08T01:14:41,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/3e114f4167344f0f9539bf434a736caf as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3e114f4167344f0f9539bf434a736caf 2024-12-08T01:14:41,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3e114f4167344f0f9539bf434a736caf, entries=100, sequenceid=253, filesize=9.5 K 2024-12-08T01:14:41,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 56865e84e13eaff78d5439ba323ab5f2 in 1387ms, sequenceid=253, compaction requested=true 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:41,223 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:41,223 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:41,224 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45910 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:41,224 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167178 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:41,224 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/A is initiating minor compaction (all files) 2024-12-08T01:14:41,224 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/B is initiating minor compaction (all files) 2024-12-08T01:14:41,224 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/A in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,224 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/B in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,224 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/1f0b4c5ea7f74dceba6b605d07d18491, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/62fdafcd37774e4b89b65b295994b4f8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3afe240033bf4b00bbab5e7d338be9c1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/632ac03afd7e42e7af2a3f71979a37c6] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=44.8 K 2024-12-08T01:14:41,224 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=163.3 K 2024-12-08T01:14:41,224 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,225 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76] 2024-12-08T01:14:41,225 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6d4f8abb2641c788ce1c62fbb17d1d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733620478181 2024-12-08T01:14:41,225 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0b4c5ea7f74dceba6b605d07d18491, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733620478184 2024-12-08T01:14:41,225 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fdafcd37774e4b89b65b295994b4f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733620478276 2024-12-08T01:14:41,225 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abfcae108b74b0bb4eefcde6d0b914e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733620478276 2024-12-08T01:14:41,226 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3afe240033bf4b00bbab5e7d338be9c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733620479330 2024-12-08T01:14:41,226 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac9be93943c442b9b17fe69eb75e4b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733620479330 2024-12-08T01:14:41,226 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 632ac03afd7e42e7af2a3f71979a37c6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620479695 2024-12-08T01:14:41,226 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d5d406283e4f48a1b2512da65c6b76, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620479695 2024-12-08T01:14:41,236 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:41,241 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208a6bfe78844aa41ef8d214ded51775905_56865e84e13eaff78d5439ba323ab5f2 store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:41,243 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:41,243 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/28b72b9cd98d4c05bb058e652f36a160 is 50, key is test_row_0/B:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:41,245 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208a6bfe78844aa41ef8d214ded51775905_56865e84e13eaff78d5439ba323ab5f2, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:41,245 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a6bfe78844aa41ef8d214ded51775905_56865e84e13eaff78d5439ba323ab5f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:41,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741977_1153 (size=4469) 2024-12-08T01:14:41,260 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#A#compaction#132 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:41,260 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/e66ad5e686ff4e60838dccea72a9051b is 175, key is test_row_0/A:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:41,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741976_1152 (size=12289) 2024-12-08T01:14:41,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741978_1154 (size=31350) 2024-12-08T01:14:41,278 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/28b72b9cd98d4c05bb058e652f36a160 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/28b72b9cd98d4c05bb058e652f36a160 2024-12-08T01:14:41,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T01:14:41,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,287 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T01:14:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:41,292 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/e66ad5e686ff4e60838dccea72a9051b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/e66ad5e686ff4e60838dccea72a9051b 2024-12-08T01:14:41,296 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/B of 56865e84e13eaff78d5439ba323ab5f2 into 28b72b9cd98d4c05bb058e652f36a160(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:41,296 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:41,296 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/B, priority=12, startTime=1733620481223; duration=0sec 2024-12-08T01:14:41,296 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:41,297 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:B 2024-12-08T01:14:41,297 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080e35c22367534ffab183126163d36ab0_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620479947/Put/seqid=0 2024-12-08T01:14:41,302 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45910 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:41,302 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/C is initiating minor compaction (all files) 2024-12-08T01:14:41,302 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/C in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:41,302 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/47b01471ff7943ba95ed3d83f1990c43, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/093ea805583a4ea0b741a8af6c808eac, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/f72e9910f03e4be2a1ae113584393fa2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3e114f4167344f0f9539bf434a736caf] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=44.8 K 2024-12-08T01:14:41,306 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b01471ff7943ba95ed3d83f1990c43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733620478184 2024-12-08T01:14:41,306 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/A of 56865e84e13eaff78d5439ba323ab5f2 into e66ad5e686ff4e60838dccea72a9051b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:41,307 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:41,307 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/A, priority=12, startTime=1733620481223; duration=0sec 2024-12-08T01:14:41,307 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:41,307 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:A 2024-12-08T01:14:41,308 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 093ea805583a4ea0b741a8af6c808eac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733620478276 2024-12-08T01:14:41,308 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f72e9910f03e4be2a1ae113584393fa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733620479330 2024-12-08T01:14:41,309 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e114f4167344f0f9539bf434a736caf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620479695 2024-12-08T01:14:41,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741979_1155 (size=12454) 2024-12-08T01:14:41,323 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:41,324 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/0efe96aa003e4308933665a5d393fe4e is 50, key is test_row_0/C:col10/1733620479695/Put/seqid=0 2024-12-08T01:14:41,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741980_1156 (size=12289) 2024-12-08T01:14:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:41,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620541548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620541549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620541562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,563 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620541652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620541662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:41,716 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080e35c22367534ffab183126163d36ab0_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080e35c22367534ffab183126163d36ab0_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/3549e290f8774ad29f1725572deb9353, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/3549e290f8774ad29f1725572deb9353 is 175, key is test_row_0/A:col10/1733620479947/Put/seqid=0 2024-12-08T01:14:41,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741981_1157 (size=31255) 2024-12-08T01:14:41,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/0efe96aa003e4308933665a5d393fe4e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/0efe96aa003e4308933665a5d393fe4e 2024-12-08T01:14:41,742 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/C of 56865e84e13eaff78d5439ba323ab5f2 into 0efe96aa003e4308933665a5d393fe4e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:41,742 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:41,742 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/C, priority=12, startTime=1733620481223; duration=0sec 2024-12-08T01:14:41,742 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:41,742 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:C 2024-12-08T01:14:41,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620541862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620541872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,121 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/3549e290f8774ad29f1725572deb9353 2024-12-08T01:14:42,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa is 50, key is test_row_0/B:col10/1733620479947/Put/seqid=0 2024-12-08T01:14:42,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741982_1158 (size=12301) 2024-12-08T01:14:42,134 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa 2024-12-08T01:14:42,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/159f1984d4774804bc8f094b24e96ccb is 50, key is test_row_0/C:col10/1733620479947/Put/seqid=0 2024-12-08T01:14:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741983_1159 (size=12301) 2024-12-08T01:14:42,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620542162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,165 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620542171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620542172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,176 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620542182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:42,546 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/159f1984d4774804bc8f094b24e96ccb 2024-12-08T01:14:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/3549e290f8774ad29f1725572deb9353 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353 2024-12-08T01:14:42,557 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353, entries=150, sequenceid=290, filesize=30.5 K 2024-12-08T01:14:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa 2024-12-08T01:14:42,566 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T01:14:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/159f1984d4774804bc8f094b24e96ccb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/159f1984d4774804bc8f094b24e96ccb 2024-12-08T01:14:42,573 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/159f1984d4774804bc8f094b24e96ccb, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T01:14:42,574 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 56865e84e13eaff78d5439ba323ab5f2 in 1288ms, sequenceid=290, compaction requested=false 2024-12-08T01:14:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-08T01:14:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-08T01:14:42,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-08T01:14:42,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2500 sec 2024-12-08T01:14:42,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2560 sec 2024-12-08T01:14:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:42,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:14:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:42,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120851b954b65cae423a8724d9dbd9bf0b7b_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:42,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741984_1160 (size=14994) 2024-12-08T01:14:42,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620482711, max timestamp=1733620482723Num uniques 859; 2024-12-08T01:14:42,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:42,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620482711, max timestamp=1733620482723Num uniques 859; 2024-12-08T01:14:42,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:42,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620482711, max timestamp=1733620482723Num uniques 859; 2024-12-08T01:14:42,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:42,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620542764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620542769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620542872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:42,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620542881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620543082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620543092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,110 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:43,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120851b954b65cae423a8724d9dbd9bf0b7b_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120851b954b65cae423a8724d9dbd9bf0b7b_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:43,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f2da1a32165e465ab1e5076b1bc77de9, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:43,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f2da1a32165e465ab1e5076b1bc77de9 is 175, key is test_row_0/A:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:43,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741985_1161 (size=39949) 2024-12-08T01:14:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620543392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620543401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f2da1a32165e465ab1e5076b1bc77de9 2024-12-08T01:14:43,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/7550e33abb7e4195bf69683e806147d0 is 50, key is test_row_0/B:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:43,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741986_1162 (size=12301) 2024-12-08T01:14:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620543592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,593 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620543902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620543912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:43,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/7550e33abb7e4195bf69683e806147d0 2024-12-08T01:14:43,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/867c3c41bf624efcaa705e7ca6b9fcb7 is 50, key is test_row_0/C:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:43,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741987_1163 (size=12301) 2024-12-08T01:14:43,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/867c3c41bf624efcaa705e7ca6b9fcb7 2024-12-08T01:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f2da1a32165e465ab1e5076b1bc77de9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9 2024-12-08T01:14:43,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9, entries=200, sequenceid=304, filesize=39.0 K 2024-12-08T01:14:43,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/7550e33abb7e4195bf69683e806147d0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/7550e33abb7e4195bf69683e806147d0 2024-12-08T01:14:43,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/7550e33abb7e4195bf69683e806147d0, entries=150, sequenceid=304, filesize=12.0 K 2024-12-08T01:14:43,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/867c3c41bf624efcaa705e7ca6b9fcb7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/867c3c41bf624efcaa705e7ca6b9fcb7 2024-12-08T01:14:43,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/867c3c41bf624efcaa705e7ca6b9fcb7, entries=150, sequenceid=304, filesize=12.0 K 2024-12-08T01:14:43,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 56865e84e13eaff78d5439ba323ab5f2 in 1278ms, sequenceid=304, compaction requested=true 2024-12-08T01:14:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:43,963 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:43,963 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:43,963 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:43,964 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102554 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:43,964 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/B is initiating minor compaction (all files) 2024-12-08T01:14:43,964 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/A is initiating minor compaction (all files) 2024-12-08T01:14:43,964 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/B in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:43,964 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/A in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:43,964 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/28b72b9cd98d4c05bb058e652f36a160, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/7550e33abb7e4195bf69683e806147d0] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=36.0 K 2024-12-08T01:14:43,964 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/e66ad5e686ff4e60838dccea72a9051b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=100.2 K 2024-12-08T01:14:43,964 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:43,964 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/e66ad5e686ff4e60838dccea72a9051b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9] 2024-12-08T01:14:43,964 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 28b72b9cd98d4c05bb058e652f36a160, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620479330 2024-12-08T01:14:43,964 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66ad5e686ff4e60838dccea72a9051b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620479330 2024-12-08T01:14:43,965 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a86eadcd9c5e4f19b0b2db68e5a2dbaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733620479947 2024-12-08T01:14:43,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3549e290f8774ad29f1725572deb9353, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733620479947 2024-12-08T01:14:43,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2da1a32165e465ab1e5076b1bc77de9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733620481536 2024-12-08T01:14:43,965 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7550e33abb7e4195bf69683e806147d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733620481536 2024-12-08T01:14:43,971 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:43,973 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:43,973 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/c189089ebc2b46809e1d0a0d73eeb976 is 50, key is test_row_0/B:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:43,974 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208d98b4f39264a454da820b8c6419f18a3_56865e84e13eaff78d5439ba323ab5f2 store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:43,976 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208d98b4f39264a454da820b8c6419f18a3_56865e84e13eaff78d5439ba323ab5f2, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:43,976 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d98b4f39264a454da820b8c6419f18a3_56865e84e13eaff78d5439ba323ab5f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:43,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741989_1165 (size=4469) 2024-12-08T01:14:44,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741988_1164 (size=12541) 2024-12-08T01:14:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T01:14:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:44,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120833608170aacf4aa092be05bf5773dec0_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620484175/Put/seqid=0 2024-12-08T01:14:44,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741990_1166 (size=14994) 2024-12-08T01:14:44,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620544200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620544206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620544311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620544322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,399 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:44,400 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/091fedcfecd044329073e3a75d1764b4 is 175, key is test_row_0/A:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741991_1167 (size=31495) 2024-12-08T01:14:44,411 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/c189089ebc2b46809e1d0a0d73eeb976 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/c189089ebc2b46809e1d0a0d73eeb976 2024-12-08T01:14:44,416 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/B of 56865e84e13eaff78d5439ba323ab5f2 into c189089ebc2b46809e1d0a0d73eeb976(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:44,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:44,416 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/B, priority=13, startTime=1733620483963; duration=0sec 2024-12-08T01:14:44,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:44,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:B 2024-12-08T01:14:44,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:44,417 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:44,417 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/C is initiating minor compaction (all files) 2024-12-08T01:14:44,418 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/C in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:44,418 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/0efe96aa003e4308933665a5d393fe4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/159f1984d4774804bc8f094b24e96ccb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/867c3c41bf624efcaa705e7ca6b9fcb7] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=36.0 K 2024-12-08T01:14:44,418 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0efe96aa003e4308933665a5d393fe4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733620479330 2024-12-08T01:14:44,419 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 159f1984d4774804bc8f094b24e96ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733620479947 2024-12-08T01:14:44,419 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 867c3c41bf624efcaa705e7ca6b9fcb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733620481536 2024-12-08T01:14:44,426 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:44,427 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/482af29b24c14c97ba645c8cd6c5d524 is 50, key is test_row_0/C:col10/1733620481548/Put/seqid=0 2024-12-08T01:14:44,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741992_1168 (size=12541) 2024-12-08T01:14:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T01:14:44,482 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:44,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-08T01:14:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T01:14:44,484 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:44,485 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:44,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620544521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620544532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T01:14:44,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:44,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120833608170aacf4aa092be05bf5773dec0_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120833608170aacf4aa092be05bf5773dec0_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:44,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/501edd23aa4e4eb1bac9cd9ce46c125d, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:44,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/501edd23aa4e4eb1bac9cd9ce46c125d is 175, key is test_row_0/A:col10/1733620484175/Put/seqid=0 2024-12-08T01:14:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741993_1169 (size=39949) 2024-12-08T01:14:44,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/501edd23aa4e4eb1bac9cd9ce46c125d 2024-12-08T01:14:44,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/b796c4d11ab4403aab71f72c5e569bea is 50, key is test_row_0/B:col10/1733620484175/Put/seqid=0 2024-12-08T01:14:44,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T01:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:44,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:44,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741994_1170 (size=12301) 2024-12-08T01:14:44,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/b796c4d11ab4403aab71f72c5e569bea 2024-12-08T01:14:44,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/e7887cbd64f14e4d88558c1b6815d161 is 50, key is test_row_0/C:col10/1733620484175/Put/seqid=0 2024-12-08T01:14:44,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741995_1171 (size=12301) 2024-12-08T01:14:44,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/e7887cbd64f14e4d88558c1b6815d161 2024-12-08T01:14:44,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/501edd23aa4e4eb1bac9cd9ce46c125d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d 2024-12-08T01:14:44,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d, entries=200, sequenceid=340, filesize=39.0 K 2024-12-08T01:14:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/b796c4d11ab4403aab71f72c5e569bea as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/b796c4d11ab4403aab71f72c5e569bea 2024-12-08T01:14:44,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/b796c4d11ab4403aab71f72c5e569bea, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T01:14:44,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/e7887cbd64f14e4d88558c1b6815d161 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/e7887cbd64f14e4d88558c1b6815d161 2024-12-08T01:14:44,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/e7887cbd64f14e4d88558c1b6815d161, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T01:14:44,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 56865e84e13eaff78d5439ba323ab5f2 in 519ms, sequenceid=340, compaction requested=false 2024-12-08T01:14:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:44,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:44,791 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:44,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120822a65cc032c24cb9ab6eb850266168e8_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620484197/Put/seqid=0 2024-12-08T01:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T01:14:44,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741996_1172 (size=12454) 2024-12-08T01:14:44,811 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/091fedcfecd044329073e3a75d1764b4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/091fedcfecd044329073e3a75d1764b4 2024-12-08T01:14:44,818 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/A of 56865e84e13eaff78d5439ba323ab5f2 into 091fedcfecd044329073e3a75d1764b4(size=30.8 K), total size for store is 69.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:44,818 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:44,818 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/A, priority=13, startTime=1733620483963; duration=0sec 2024-12-08T01:14:44,818 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:44,819 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:A 2024-12-08T01:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:44,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:44,839 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/482af29b24c14c97ba645c8cd6c5d524 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/482af29b24c14c97ba645c8cd6c5d524 2024-12-08T01:14:44,845 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/C of 56865e84e13eaff78d5439ba323ab5f2 into 482af29b24c14c97ba645c8cd6c5d524(size=12.2 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:44,845 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:44,845 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/C, priority=13, startTime=1733620483963; duration=0sec 2024-12-08T01:14:44,845 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:44,845 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:C 2024-12-08T01:14:44,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620484864, max timestamp=1733620484876Num uniques 859; 2024-12-08T01:14:44,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620484864, max timestamp=1733620484876Num uniques 859; 2024-12-08T01:14:44,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:44,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:44,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620484864, max timestamp=1733620484876Num uniques 859; 2024-12-08T01:14:44,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:44,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620544908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620544911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620544922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,929 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620544932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:44,937 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620545022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:45,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620545022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T01:14:45,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:45,208 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120822a65cc032c24cb9ab6eb850266168e8_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120822a65cc032c24cb9ab6eb850266168e8_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f341848d248440aba641f82cb1e6fed4, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:45,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f341848d248440aba641f82cb1e6fed4 is 175, key is test_row_0/A:col10/1733620484197/Put/seqid=0 2024-12-08T01:14:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741997_1173 (size=31255) 2024-12-08T01:14:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620545232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:45,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620545232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620545541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620545541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:45,614 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f341848d248440aba641f82cb1e6fed4 2024-12-08T01:14:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T01:14:45,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/e07ee19af63d44689faf162d9d8fcf30 is 50, key is test_row_0/B:col10/1733620484197/Put/seqid=0 2024-12-08T01:14:45,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741998_1174 (size=12301) 2024-12-08T01:14:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620546051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620546051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:46,069 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/e07ee19af63d44689faf162d9d8fcf30 2024-12-08T01:14:46,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/043340db782048c886ad0cb72173bff8 is 50, key is test_row_0/C:col10/1733620484197/Put/seqid=0 2024-12-08T01:14:46,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741999_1175 (size=12301) 2024-12-08T01:14:46,083 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/043340db782048c886ad0cb72173bff8 2024-12-08T01:14:46,091 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T01:14:46,091 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T01:14:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f341848d248440aba641f82cb1e6fed4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4 2024-12-08T01:14:46,099 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4, entries=150, sequenceid=351, filesize=30.5 K 2024-12-08T01:14:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/e07ee19af63d44689faf162d9d8fcf30 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e07ee19af63d44689faf162d9d8fcf30 2024-12-08T01:14:46,105 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e07ee19af63d44689faf162d9d8fcf30, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T01:14:46,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/043340db782048c886ad0cb72173bff8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/043340db782048c886ad0cb72173bff8 2024-12-08T01:14:46,111 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/043340db782048c886ad0cb72173bff8, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T01:14:46,112 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 56865e84e13eaff78d5439ba323ab5f2 in 1321ms, sequenceid=351, compaction requested=true 2024-12-08T01:14:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-08T01:14:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-08T01:14:46,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-08T01:14:46,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-12-08T01:14:46,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6330 sec 2024-12-08T01:14:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T01:14:46,632 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:46,632 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-08T01:14:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T01:14:46,634 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:46,635 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:46,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T01:14:46,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:46,787 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c36c3d2db2de4fa498e5617d63512d7b_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620484910/Put/seqid=0 2024-12-08T01:14:46,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742000_1176 (size=12454) 2024-12-08T01:14:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:46,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T01:14:46,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620546957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620546959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620547062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620547062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,066 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,066 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620547072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620547072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:47,208 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c36c3d2db2de4fa498e5617d63512d7b_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c36c3d2db2de4fa498e5617d63512d7b_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/c6e34c3ccc364f25b2a44916bbb5053f, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/c6e34c3ccc364f25b2a44916bbb5053f is 175, key is test_row_0/A:col10/1733620484910/Put/seqid=0 2024-12-08T01:14:47,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742001_1177 (size=31255) 2024-12-08T01:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T01:14:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620547282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620547282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620547592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620547592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,615 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/c6e34c3ccc364f25b2a44916bbb5053f 2024-12-08T01:14:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/2470b87ffd854b0a81d607616aad3d7f is 50, key is test_row_0/B:col10/1733620484910/Put/seqid=0 2024-12-08T01:14:47,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620547622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:47,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742002_1178 (size=12301) 2024-12-08T01:14:47,626 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T01:14:48,027 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/2470b87ffd854b0a81d607616aad3d7f 2024-12-08T01:14:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/d764d41b07924d6bbef503a67fa8d13d is 50, key is test_row_0/C:col10/1733620484910/Put/seqid=0 2024-12-08T01:14:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742003_1179 (size=12301) 2024-12-08T01:14:48,044 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/d764d41b07924d6bbef503a67fa8d13d 2024-12-08T01:14:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/c6e34c3ccc364f25b2a44916bbb5053f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f 2024-12-08T01:14:48,052 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f, entries=150, sequenceid=388, filesize=30.5 K 2024-12-08T01:14:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/2470b87ffd854b0a81d607616aad3d7f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/2470b87ffd854b0a81d607616aad3d7f 2024-12-08T01:14:48,057 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/2470b87ffd854b0a81d607616aad3d7f, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T01:14:48,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/d764d41b07924d6bbef503a67fa8d13d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/d764d41b07924d6bbef503a67fa8d13d 2024-12-08T01:14:48,063 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/d764d41b07924d6bbef503a67fa8d13d, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T01:14:48,064 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 56865e84e13eaff78d5439ba323ab5f2 in 1277ms, sequenceid=388, compaction requested=true 2024-12-08T01:14:48,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:48,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:48,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-08T01:14:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-08T01:14:48,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-08T01:14:48,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-12-08T01:14:48,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4350 sec 2024-12-08T01:14:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:48,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:14:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:48,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083996ae5bab65475da0170140f3e169d9_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:48,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742004_1180 (size=14994) 2024-12-08T01:14:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620488121, max timestamp=1733620488136Num uniques 859; 2024-12-08T01:14:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620488121, max timestamp=1733620488136Num uniques 859; 2024-12-08T01:14:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620488121, max timestamp=1733620488136Num uniques 859; 2024-12-08T01:14:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:48,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620548169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620548170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620548282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620548282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:48,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083996ae5bab65475da0170140f3e169d9_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083996ae5bab65475da0170140f3e169d9_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:48,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/b617a37c866243e28fc4602c85546e7e, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:48,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/b617a37c866243e28fc4602c85546e7e is 175, key is test_row_0/A:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:48,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742005_1181 (size=39949) 2024-12-08T01:14:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T01:14:48,783 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:48,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-08T01:14:48,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:48,788 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:48,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:48,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620548803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620548803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:48,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/b617a37c866243e28fc4602c85546e7e 2024-12-08T01:14:48,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/dfb2d4b8765b46c2a4513d52f736dd2a is 50, key is test_row_0/B:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:48,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T01:14:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:48,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:48,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:48,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742006_1182 (size=12301) 2024-12-08T01:14:48,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/dfb2d4b8765b46c2a4513d52f736dd2a 2024-12-08T01:14:48,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/8a975bdef8d54f26a8cfdc60160fb154 is 50, key is test_row_0/C:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:49,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742007_1183 (size=12301) 2024-12-08T01:14:49,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/8a975bdef8d54f26a8cfdc60160fb154 2024-12-08T01:14:49,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/b617a37c866243e28fc4602c85546e7e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e 2024-12-08T01:14:49,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e, entries=200, sequenceid=399, filesize=39.0 K 2024-12-08T01:14:49,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/dfb2d4b8765b46c2a4513d52f736dd2a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/dfb2d4b8765b46c2a4513d52f736dd2a 2024-12-08T01:14:49,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/dfb2d4b8765b46c2a4513d52f736dd2a, entries=150, sequenceid=399, filesize=12.0 K 2024-12-08T01:14:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/8a975bdef8d54f26a8cfdc60160fb154 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/8a975bdef8d54f26a8cfdc60160fb154 2024-12-08T01:14:49,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/8a975bdef8d54f26a8cfdc60160fb154, entries=150, sequenceid=399, filesize=12.0 K 2024-12-08T01:14:49,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 56865e84e13eaff78d5439ba323ab5f2 in 940ms, sequenceid=399, compaction requested=true 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:49,044 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:49,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:14:49,046 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173903 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T01:14:49,046 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/A is initiating minor compaction (all files) 2024-12-08T01:14:49,046 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/A in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,046 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/091fedcfecd044329073e3a75d1764b4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=169.8 K 2024-12-08T01:14:49,046 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,046 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/091fedcfecd044329073e3a75d1764b4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e] 2024-12-08T01:14:49,047 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T01:14:49,047 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 091fedcfecd044329073e3a75d1764b4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733620481536 2024-12-08T01:14:49,048 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 501edd23aa4e4eb1bac9cd9ce46c125d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620482748 2024-12-08T01:14:49,048 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f341848d248440aba641f82cb1e6fed4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733620484186 2024-12-08T01:14:49,049 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e34c3ccc364f25b2a44916bbb5053f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733620484896 2024-12-08T01:14:49,049 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61745 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T01:14:49,049 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/B is initiating minor compaction (all files) 2024-12-08T01:14:49,049 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/B in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,049 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/c189089ebc2b46809e1d0a0d73eeb976, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/b796c4d11ab4403aab71f72c5e569bea, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e07ee19af63d44689faf162d9d8fcf30, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/2470b87ffd854b0a81d607616aad3d7f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/dfb2d4b8765b46c2a4513d52f736dd2a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=60.3 K 2024-12-08T01:14:49,050 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b617a37c866243e28fc4602c85546e7e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620486943 2024-12-08T01:14:49,050 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c189089ebc2b46809e1d0a0d73eeb976, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733620481536 2024-12-08T01:14:49,051 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting b796c4d11ab4403aab71f72c5e569bea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620482748 2024-12-08T01:14:49,051 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e07ee19af63d44689faf162d9d8fcf30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733620484186 2024-12-08T01:14:49,052 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2470b87ffd854b0a81d607616aad3d7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733620484896 2024-12-08T01:14:49,052 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfb2d4b8765b46c2a4513d52f736dd2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620486943 2024-12-08T01:14:49,065 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:49,071 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:49,072 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/366ccb5be2084fe588d00f542a130b7d is 50, key is test_row_0/B:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:49,077 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208fef0e1de3f1640a58c98185d4a951e6e_56865e84e13eaff78d5439ba323ab5f2 store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:49,080 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208fef0e1de3f1640a58c98185d4a951e6e_56865e84e13eaff78d5439ba323ab5f2, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:49,080 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fef0e1de3f1640a58c98185d4a951e6e_56865e84e13eaff78d5439ba323ab5f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:49,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742008_1184 (size=12711) 2024-12-08T01:14:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:49,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T01:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:49,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742009_1185 (size=4469) 2024-12-08T01:14:49,095 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/366ccb5be2084fe588d00f542a130b7d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/366ccb5be2084fe588d00f542a130b7d 2024-12-08T01:14:49,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T01:14:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,098 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#A#compaction#156 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,099 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/98d88d760d3649758432def0d9a25c4d is 175, key is test_row_0/A:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:49,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120871b11870429a4dd2bcbdcd9641a9e8da_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620488168/Put/seqid=0 2024-12-08T01:14:49,106 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/B of 56865e84e13eaff78d5439ba323ab5f2 into 366ccb5be2084fe588d00f542a130b7d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:49,106 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:49,106 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/B, priority=11, startTime=1733620489044; duration=0sec 2024-12-08T01:14:49,106 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:49,106 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:B 2024-12-08T01:14:49,107 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T01:14:49,111 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61745 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T01:14:49,111 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/C is initiating minor compaction (all files) 2024-12-08T01:14:49,111 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/C in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,111 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/482af29b24c14c97ba645c8cd6c5d524, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/e7887cbd64f14e4d88558c1b6815d161, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/043340db782048c886ad0cb72173bff8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/d764d41b07924d6bbef503a67fa8d13d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/8a975bdef8d54f26a8cfdc60160fb154] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=60.3 K 2024-12-08T01:14:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620549107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,112 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 482af29b24c14c97ba645c8cd6c5d524, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733620481536 2024-12-08T01:14:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,113 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7887cbd64f14e4d88558c1b6815d161, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620482748 2024-12-08T01:14:49,113 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 043340db782048c886ad0cb72173bff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733620484186 2024-12-08T01:14:49,113 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting d764d41b07924d6bbef503a67fa8d13d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733620484896 2024-12-08T01:14:49,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620549112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,114 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a975bdef8d54f26a8cfdc60160fb154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620486943 2024-12-08T01:14:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742010_1186 (size=31665) 2024-12-08T01:14:49,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742011_1187 (size=14994) 2024-12-08T01:14:49,120 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:49,124 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120871b11870429a4dd2bcbdcd9641a9e8da_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120871b11870429a4dd2bcbdcd9641a9e8da_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:49,127 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:49,128 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/9867589da4b34bd78ade5a3e5036d252 is 50, key is test_row_0/C:col10/1733620486958/Put/seqid=0 2024-12-08T01:14:49,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/14c218da13dc4cd7aaedbf36a300d741, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:49,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/14c218da13dc4cd7aaedbf36a300d741 is 175, key is test_row_0/A:col10/1733620488168/Put/seqid=0 2024-12-08T01:14:49,128 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/98d88d760d3649758432def0d9a25c4d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/98d88d760d3649758432def0d9a25c4d 2024-12-08T01:14:49,135 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/A of 56865e84e13eaff78d5439ba323ab5f2 into 98d88d760d3649758432def0d9a25c4d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:49,135 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:49,135 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/A, priority=11, startTime=1733620489044; duration=0sec 2024-12-08T01:14:49,135 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:49,135 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:A 2024-12-08T01:14:49,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742012_1188 (size=39949) 2024-12-08T01:14:49,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/14c218da13dc4cd7aaedbf36a300d741 2024-12-08T01:14:49,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742013_1189 (size=12711) 2024-12-08T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/456126ef7fc34b088c229a0149423a09 is 50, key is test_row_0/B:col10/1733620488168/Put/seqid=0 2024-12-08T01:14:49,157 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/9867589da4b34bd78ade5a3e5036d252 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/9867589da4b34bd78ade5a3e5036d252 2024-12-08T01:14:49,166 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/C of 56865e84e13eaff78d5439ba323ab5f2 into 9867589da4b34bd78ade5a3e5036d252(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:49,166 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:49,166 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/C, priority=11, startTime=1733620489044; duration=0sec 2024-12-08T01:14:49,166 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:49,167 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:C 2024-12-08T01:14:49,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742014_1190 (size=12301) 2024-12-08T01:14:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620549221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620549222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T01:14:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620549322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620549344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T01:14:49,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:49,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:49,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620549431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620549431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T01:14:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:49,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/456126ef7fc34b088c229a0149423a09 2024-12-08T01:14:49,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/c46fd6b6fbe349bb8ec1e2b41b757a28 is 50, key is test_row_0/C:col10/1733620488168/Put/seqid=0 2024-12-08T01:14:49,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742015_1191 (size=12301) 2024-12-08T01:14:49,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/c46fd6b6fbe349bb8ec1e2b41b757a28 2024-12-08T01:14:49,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/14c218da13dc4cd7aaedbf36a300d741 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741 2024-12-08T01:14:49,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741, entries=200, sequenceid=436, filesize=39.0 K 2024-12-08T01:14:49,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/456126ef7fc34b088c229a0149423a09 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/456126ef7fc34b088c229a0149423a09 2024-12-08T01:14:49,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/456126ef7fc34b088c229a0149423a09, entries=150, sequenceid=436, filesize=12.0 K 2024-12-08T01:14:49,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/c46fd6b6fbe349bb8ec1e2b41b757a28 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/c46fd6b6fbe349bb8ec1e2b41b757a28 2024-12-08T01:14:49,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/c46fd6b6fbe349bb8ec1e2b41b757a28, entries=150, sequenceid=436, filesize=12.0 K 2024-12-08T01:14:49,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 56865e84e13eaff78d5439ba323ab5f2 in 530ms, sequenceid=436, compaction requested=false 2024-12-08T01:14:49,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:49,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T01:14:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:49,709 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-08T01:14:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:49,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:49,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:49,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:49,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:49,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:49,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208169ed9b97cc84e1c83fb4376e6e3e8b1_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620489095/Put/seqid=0 2024-12-08T01:14:49,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742016_1192 (size=12454) 2024-12-08T01:14:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:49,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620489761, max timestamp=1733620489787Num uniques 859; 2024-12-08T01:14:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620489761, max timestamp=1733620489787Num uniques 859; 2024-12-08T01:14:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:49,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620489761, max timestamp=1733620489787Num uniques 859; 2024-12-08T01:14:49,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:14:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620489825, max timestamp=1733620489854Num uniques 1018; 2024-12-08T01:14:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T01:14:49,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620489825, max timestamp=1733620489854Num uniques 1018; 2024-12-08T01:14:49,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T01:14:49,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620489825, max timestamp=1733620489854Num uniques 1018; 2024-12-08T01:14:49,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T01:14:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620549934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620549936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620550052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620550052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:50,148 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208169ed9b97cc84e1c83fb4376e6e3e8b1_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208169ed9b97cc84e1c83fb4376e6e3e8b1_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:50,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/cf39242e052040f687efb3b9262484cc, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:50,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/cf39242e052040f687efb3b9262484cc is 175, key is test_row_0/A:col10/1733620489095/Put/seqid=0 2024-12-08T01:14:50,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742017_1193 (size=31255) 2024-12-08T01:14:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620550262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620550272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620550353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,355 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620550368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,369 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620550572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620550582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:50,597 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/cf39242e052040f687efb3b9262484cc 2024-12-08T01:14:50,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/bd2a641e9678488d96ee5d3dbb57bcb0 is 50, key is test_row_0/B:col10/1733620489095/Put/seqid=0 2024-12-08T01:14:50,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742018_1194 (size=12301) 2024-12-08T01:14:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:51,071 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/bd2a641e9678488d96ee5d3dbb57bcb0 2024-12-08T01:14:51,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/65ff5d023e3f413aa5d00cddc51be467 is 50, key is test_row_0/C:col10/1733620489095/Put/seqid=0 2024-12-08T01:14:51,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742019_1195 (size=12301) 2024-12-08T01:14:51,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620551082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:51,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:51,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:51,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:51,089 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/65ff5d023e3f413aa5d00cddc51be467 2024-12-08T01:14:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/cf39242e052040f687efb3b9262484cc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc 2024-12-08T01:14:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620551092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:51,097 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc, entries=150, sequenceid=448, filesize=30.5 K 2024-12-08T01:14:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/bd2a641e9678488d96ee5d3dbb57bcb0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/bd2a641e9678488d96ee5d3dbb57bcb0 2024-12-08T01:14:51,102 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/bd2a641e9678488d96ee5d3dbb57bcb0, entries=150, sequenceid=448, filesize=12.0 K 2024-12-08T01:14:51,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/65ff5d023e3f413aa5d00cddc51be467 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/65ff5d023e3f413aa5d00cddc51be467 2024-12-08T01:14:51,107 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/65ff5d023e3f413aa5d00cddc51be467, entries=150, sequenceid=448, filesize=12.0 K 2024-12-08T01:14:51,108 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 56865e84e13eaff78d5439ba323ab5f2 in 1399ms, sequenceid=448, compaction requested=true 2024-12-08T01:14:51,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:51,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:51,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-08T01:14:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-08T01:14:51,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-08T01:14:51,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-12-08T01:14:51,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.3280 sec 2024-12-08T01:14:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:52,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-12-08T01:14:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:14:52,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a8fb2af17cd74a8ab7e5cca7e52e8039_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:52,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742020_1196 (size=14994) 2024-12-08T01:14:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620552117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620552121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620552231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620552231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620552361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,365 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620552392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,395 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620552442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620552442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:52,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a8fb2af17cd74a8ab7e5cca7e52e8039_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a8fb2af17cd74a8ab7e5cca7e52e8039_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:52,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/755a092ce8c942cea5acdb338c0f0c4b, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:52,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/755a092ce8c942cea5acdb338c0f0c4b is 175, key is test_row_0/A:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:52,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742021_1197 (size=39949) 2024-12-08T01:14:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620552752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620552752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:52,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=502, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/755a092ce8c942cea5acdb338c0f0c4b 2024-12-08T01:14:52,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/3dc8a4c6f4474ac78cb3b85e27627609 is 50, key is test_row_0/B:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:52,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742022_1198 (size=12301) 2024-12-08T01:14:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T01:14:52,942 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:52,942 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-08T01:14:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:52,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:52,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:52,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:53,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T01:14:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T01:14:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:53,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620553262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:53,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620553262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:53,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/3dc8a4c6f4474ac78cb3b85e27627609 2024-12-08T01:14:53,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/450992920282413a9601188d1539caf2 is 50, key is test_row_0/C:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:53,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742023_1199 (size=12301) 2024-12-08T01:14:53,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T01:14:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T01:14:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:53,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T01:14:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:14:53,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/450992920282413a9601188d1539caf2 2024-12-08T01:14:53,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/755a092ce8c942cea5acdb338c0f0c4b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b 2024-12-08T01:14:53,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b, entries=200, sequenceid=502, filesize=39.0 K 2024-12-08T01:14:53,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/3dc8a4c6f4474ac78cb3b85e27627609 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3dc8a4c6f4474ac78cb3b85e27627609 2024-12-08T01:14:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3dc8a4c6f4474ac78cb3b85e27627609, entries=150, sequenceid=502, filesize=12.0 K 2024-12-08T01:14:53,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/450992920282413a9601188d1539caf2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/450992920282413a9601188d1539caf2 2024-12-08T01:14:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/450992920282413a9601188d1539caf2, entries=150, sequenceid=502, filesize=12.0 K 2024-12-08T01:14:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=33.54 KB/34350 for 56865e84e13eaff78d5439ba323ab5f2 in 1668ms, sequenceid=502, compaction requested=true 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:53,771 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:53,771 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,773 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:53,773 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/A is initiating minor compaction (all files) 2024-12-08T01:14:53,773 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/A in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,773 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/98d88d760d3649758432def0d9a25c4d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=139.5 K 2024-12-08T01:14:53,773 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,773 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/98d88d760d3649758432def0d9a25c4d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b] 2024-12-08T01:14:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,774 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:53,774 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/B is initiating minor compaction (all files) 2024-12-08T01:14:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,774 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d88d760d3649758432def0d9a25c4d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620486943 2024-12-08T01:14:53,774 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/B in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,774 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/366ccb5be2084fe588d00f542a130b7d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/456126ef7fc34b088c229a0149423a09, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/bd2a641e9678488d96ee5d3dbb57bcb0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3dc8a4c6f4474ac78cb3b85e27627609] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=48.5 K 2024-12-08T01:14:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,774 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c218da13dc4cd7aaedbf36a300d741, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733620488167 2024-12-08T01:14:53,774 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 366ccb5be2084fe588d00f542a130b7d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620486943 2024-12-08T01:14:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,775 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf39242e052040f687efb3b9262484cc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733620489095 2024-12-08T01:14:53,775 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 456126ef7fc34b088c229a0149423a09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733620488167 2024-12-08T01:14:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,775 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2a641e9678488d96ee5d3dbb57bcb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733620489095 2024-12-08T01:14:53,775 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755a092ce8c942cea5acdb338c0f0c4b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733620489921 2024-12-08T01:14:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,776 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc8a4c6f4474ac78cb3b85e27627609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733620489921 2024-12-08T01:14:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,785 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,787 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/85ba503c1e1248afa73ef343e5f28aaa is 50, key is test_row_0/B:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:53,787 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087d01504bb1c14fc9a96fbac57cc41824_56865e84e13eaff78d5439ba323ab5f2 store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,790 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087d01504bb1c14fc9a96fbac57cc41824_56865e84e13eaff78d5439ba323ab5f2, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,790 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087d01504bb1c14fc9a96fbac57cc41824_56865e84e13eaff78d5439ba323ab5f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742024_1200 (size=12847) 2024-12-08T01:14:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742025_1201 (size=4469) 2024-12-08T01:14:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,834 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#A#compaction#168 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,835 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/d9b6c1b010c2463ea1e31b93a7d17e27 is 175, key is test_row_0/A:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742026_1202 (size=31801) 2024-12-08T01:14:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,854 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/d9b6c1b010c2463ea1e31b93a7d17e27 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/d9b6c1b010c2463ea1e31b93a7d17e27 2024-12-08T01:14:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T01:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,861 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/A of 56865e84e13eaff78d5439ba323ab5f2 into d9b6c1b010c2463ea1e31b93a7d17e27(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:53,861 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:53,861 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T01:14:53,861 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/A, priority=12, startTime=1733620493771; duration=0sec 2024-12-08T01:14:53,862 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:53,862 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:A 2024-12-08T01:14:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:53,862 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:14:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,863 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:14:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,864 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/C is initiating minor compaction (all files) 2024-12-08T01:14:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,864 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/C in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:53,864 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/9867589da4b34bd78ade5a3e5036d252, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/c46fd6b6fbe349bb8ec1e2b41b757a28, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/65ff5d023e3f413aa5d00cddc51be467, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/450992920282413a9601188d1539caf2] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=48.5 K 2024-12-08T01:14:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,865 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9867589da4b34bd78ade5a3e5036d252, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620486943 2024-12-08T01:14:53,865 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46fd6b6fbe349bb8ec1e2b41b757a28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733620488167 2024-12-08T01:14:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,866 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ff5d023e3f413aa5d00cddc51be467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733620489095 2024-12-08T01:14:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,866 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450992920282413a9601188d1539caf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733620489921 2024-12-08T01:14:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084c3fdb5c31274cb1859f6e3b11c83594_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_1/A:col10/1733620492106/Put/seqid=0 2024-12-08T01:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,879 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,879 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/b708a78ef90c499f9959bd44631a3f98 is 50, key is test_row_0/C:col10/1733620492102/Put/seqid=0 2024-12-08T01:14:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742028_1204 (size=12847) 2024-12-08T01:14:53,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742027_1203 (size=9914) 2024-12-08T01:14:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,237 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/85ba503c1e1248afa73ef343e5f28aaa as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/85ba503c1e1248afa73ef343e5f28aaa 2024-12-08T01:14:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,243 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/B of 56865e84e13eaff78d5439ba323ab5f2 into 85ba503c1e1248afa73ef343e5f28aaa(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:54,243 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:54,243 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/B, priority=12, startTime=1733620493771; duration=0sec 2024-12-08T01:14:54,243 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:54,243 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:B 2024-12-08T01:14:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:54,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:54,318 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084c3fdb5c31274cb1859f6e3b11c83594_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084c3fdb5c31274cb1859f6e3b11c83594_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:54,321 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/b708a78ef90c499f9959bd44631a3f98 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b708a78ef90c499f9959bd44631a3f98 2024-12-08T01:14:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f12a52201fb34edbb9a05cc3504268f1, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f12a52201fb34edbb9a05cc3504268f1 is 175, key is test_row_1/A:col10/1733620492106/Put/seqid=0 2024-12-08T01:14:54,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742029_1205 (size=22561) 2024-12-08T01:14:54,327 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=511, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f12a52201fb34edbb9a05cc3504268f1 2024-12-08T01:14:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/74532796b1614206a2b538f847f5e266 is 50, key is test_row_1/B:col10/1733620492106/Put/seqid=0 2024-12-08T01:14:54,338 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/C of 56865e84e13eaff78d5439ba323ab5f2 into b708a78ef90c499f9959bd44631a3f98(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:54,338 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:54,338 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/C, priority=12, startTime=1733620493771; duration=0sec 2024-12-08T01:14:54,338 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:54,338 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:C 2024-12-08T01:14:54,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620494311, max timestamp=1733620494336Num uniques 859; 2024-12-08T01:14:54,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:14:54,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733620494331, max timestamp=1733620494336Num uniques 859; 2024-12-08T01:14:54,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T01:14:54,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733620494331, max timestamp=1733620494336Num uniques 859; 2024-12-08T01:14:54,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T01:14:54,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742030_1206 (size=9857) 2024-12-08T01:14:54,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620554378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:54,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620494366, max timestamp=1733620494378Num uniques 1118; 2024-12-08T01:14:54,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-08T01:14:54,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620494366, max timestamp=1733620494378Num uniques 1009; 2024-12-08T01:14:54,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-12-08T01:14:54,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620494366, max timestamp=1733620494378Num uniques 1009; 2024-12-08T01:14:54,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-12-08T01:14:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620554432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620554492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:54,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620554542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620554701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620554752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:54,754 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/74532796b1614206a2b538f847f5e266 2024-12-08T01:14:54,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/899541a7276d404296c497bb668a5fd4 is 50, key is test_row_1/C:col10/1733620492106/Put/seqid=0 2024-12-08T01:14:54,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742031_1207 (size=9857) 2024-12-08T01:14:54,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:14:55,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620555012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:55,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620555061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:55,165 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/899541a7276d404296c497bb668a5fd4 2024-12-08T01:14:55,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/f12a52201fb34edbb9a05cc3504268f1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1 2024-12-08T01:14:55,173 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1, entries=100, sequenceid=511, filesize=22.0 K 2024-12-08T01:14:55,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/74532796b1614206a2b538f847f5e266 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/74532796b1614206a2b538f847f5e266 2024-12-08T01:14:55,178 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/74532796b1614206a2b538f847f5e266, entries=100, sequenceid=511, filesize=9.6 K 2024-12-08T01:14:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/899541a7276d404296c497bb668a5fd4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/899541a7276d404296c497bb668a5fd4 2024-12-08T01:14:55,184 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/899541a7276d404296c497bb668a5fd4, entries=100, sequenceid=511, filesize=9.6 K 2024-12-08T01:14:55,185 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 56865e84e13eaff78d5439ba323ab5f2 in 1324ms, sequenceid=511, compaction requested=false 2024-12-08T01:14:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-08T01:14:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-08T01:14:55,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-08T01:14:55,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-12-08T01:14:55,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2470 sec 2024-12-08T01:14:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:55,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-12-08T01:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:55,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086a569a793edf486bb32eb25946f190d2_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:55,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742032_1208 (size=14994) 2024-12-08T01:14:55,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620555541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620555571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:55,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620555652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:55,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620555862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:55,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:55,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086a569a793edf486bb32eb25946f190d2_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086a569a793edf486bb32eb25946f190d2_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:55,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/9cc8f79720dd46cfa9f8764d61a55492, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:55,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/9cc8f79720dd46cfa9f8764d61a55492 is 175, key is test_row_0/A:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:55,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742033_1209 (size=39949) 2024-12-08T01:14:56,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620556172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:56,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40716 deadline: 1733620556381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:56,383 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:56,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=568, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/9cc8f79720dd46cfa9f8764d61a55492 2024-12-08T01:14:56,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/13a73955fd4c43059d0119fefec4c4ed is 50, key is test_row_0/B:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742034_1210 (size=12301) 2024-12-08T01:14:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40702 deadline: 1733620556422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:56,424 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620556591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:56,593 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:56,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620556682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:56,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:56,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/13a73955fd4c43059d0119fefec4c4ed 2024-12-08T01:14:56,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/6f5bf46875724e07ab48c804e083d134 is 50, key is test_row_0/C:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:56,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742035_1211 (size=12301) 2024-12-08T01:14:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T01:14:57,112 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:57,112 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:14:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-08T01:14:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T01:14:57,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:14:57,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:14:57,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:14:57,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/6f5bf46875724e07ab48c804e083d134 2024-12-08T01:14:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/9cc8f79720dd46cfa9f8764d61a55492 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492 2024-12-08T01:14:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T01:14:57,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492, entries=200, sequenceid=568, filesize=39.0 K 2024-12-08T01:14:57,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/13a73955fd4c43059d0119fefec4c4ed as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/13a73955fd4c43059d0119fefec4c4ed 2024-12-08T01:14:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/13a73955fd4c43059d0119fefec4c4ed, entries=150, sequenceid=568, filesize=12.0 K 2024-12-08T01:14:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/6f5bf46875724e07ab48c804e083d134 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/6f5bf46875724e07ab48c804e083d134 2024-12-08T01:14:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/6f5bf46875724e07ab48c804e083d134, entries=150, sequenceid=568, filesize=12.0 K 2024-12-08T01:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for 56865e84e13eaff78d5439ba323ab5f2 in 1711ms, sequenceid=568, compaction requested=true 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,236 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:14:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:57,236 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:14:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,237 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:57,237 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:57,237 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/B is initiating minor compaction (all files) 2024-12-08T01:14:57,237 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/A is initiating minor compaction (all files) 2024-12-08T01:14:57,237 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/B in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:57,237 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/A in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,237 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/85ba503c1e1248afa73ef343e5f28aaa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/74532796b1614206a2b538f847f5e266, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/13a73955fd4c43059d0119fefec4c4ed] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=34.2 K 2024-12-08T01:14:57,237 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/d9b6c1b010c2463ea1e31b93a7d17e27, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=92.1 K 2024-12-08T01:14:57,237 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:57,237 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/d9b6c1b010c2463ea1e31b93a7d17e27, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492] 2024-12-08T01:14:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,238 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ba503c1e1248afa73ef343e5f28aaa, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733620489921 2024-12-08T01:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,238 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 74532796b1614206a2b538f847f5e266, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733620492106 2024-12-08T01:14:57,238 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a73955fd4c43059d0119fefec4c4ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1733620494426 2024-12-08T01:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,240 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9b6c1b010c2463ea1e31b93a7d17e27, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733620489921 2024-12-08T01:14:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,240 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12a52201fb34edbb9a05cc3504268f1, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733620492106 2024-12-08T01:14:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,241 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc8f79720dd46cfa9f8764d61a55492, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1733620494397 2024-12-08T01:14:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,246 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,247 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/f29d05145ec24fc5beaab68d16131f83 is 50, key is test_row_0/B:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,247 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,250 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208e2020eee9a9246d9b8cb7a83412853e4_56865e84e13eaff78d5439ba323ab5f2 store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,252 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208e2020eee9a9246d9b8cb7a83412853e4_56865e84e13eaff78d5439ba323ab5f2, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:57,252 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e2020eee9a9246d9b8cb7a83412853e4_56865e84e13eaff78d5439ba323ab5f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742036_1212 (size=12949) 2024-12-08T01:14:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,262 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/f29d05145ec24fc5beaab68d16131f83 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/f29d05145ec24fc5beaab68d16131f83 2024-12-08T01:14:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T01:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,266 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/B of 56865e84e13eaff78d5439ba323ab5f2 into f29d05145ec24fc5beaab68d16131f83(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:57,266 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T01:14:57,267 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:57,267 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/B, priority=13, startTime=1733620497236; duration=0sec 2024-12-08T01:14:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,267 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:14:57,267 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:B 2024-12-08T01:14:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:57,267 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:14:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:14:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,269 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:14:57,269 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/C is initiating minor compaction (all files) 2024-12-08T01:14:57,269 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/C in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:57,269 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b708a78ef90c499f9959bd44631a3f98, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/899541a7276d404296c497bb668a5fd4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/6f5bf46875724e07ab48c804e083d134] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=34.2 K 2024-12-08T01:14:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,270 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b708a78ef90c499f9959bd44631a3f98, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733620489921 2024-12-08T01:14:57,270 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 899541a7276d404296c497bb668a5fd4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733620492106 2024-12-08T01:14:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,271 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5bf46875724e07ab48c804e083d134, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1733620494426 2024-12-08T01:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742037_1213 (size=4469) 2024-12-08T01:14:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,288 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,289 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/a9d2e0d0d2a7475d9aa32539529e31e5 is 50, key is test_row_0/C:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:57,291 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#A#compaction#178 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:14:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,292 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/7c17badf13374154ad6e00856664aec4 is 175, key is test_row_0/A:col10/1733620494430/Put/seqid=0 2024-12-08T01:14:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208632e41fa09f2407dad372b8c311c7ad1_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620495540/Put/seqid=0 2024-12-08T01:14:57,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742038_1214 (size=12949) 2024-12-08T01:14:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,305 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/a9d2e0d0d2a7475d9aa32539529e31e5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/a9d2e0d0d2a7475d9aa32539529e31e5 2024-12-08T01:14:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742039_1215 (size=31903) 2024-12-08T01:14:57,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742040_1216 (size=9914) 2024-12-08T01:14:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,310 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/C of 56865e84e13eaff78d5439ba323ab5f2 into a9d2e0d0d2a7475d9aa32539529e31e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:57,310 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:57,310 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/C, priority=13, startTime=1733620497236; duration=0sec 2024-12-08T01:14:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,312 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:57,312 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:C 2024-12-08T01:14:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T01:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:57,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. as already flushing 2024-12-08T01:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:57,719 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208632e41fa09f2407dad372b8c311c7ad1_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208632e41fa09f2407dad372b8c311c7ad1_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:57,721 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/7c17badf13374154ad6e00856664aec4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/7c17badf13374154ad6e00856664aec4 2024-12-08T01:14:57,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620497694, max timestamp=1733620497717Num uniques 859; 2024-12-08T01:14:57,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:57,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620497694, max timestamp=1733620497717Num uniques 859; 2024-12-08T01:14:57,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:14:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5e58cdef74d644c39b0953c4cd87a94f, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5e58cdef74d644c39b0953c4cd87a94f is 175, key is test_row_0/A:col10/1733620495540/Put/seqid=0 2024-12-08T01:14:57,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733620497694, max timestamp=1733620497717Num uniques 859; 2024-12-08T01:14:57,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T01:14:57,736 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/A of 56865e84e13eaff78d5439ba323ab5f2 into 7c17badf13374154ad6e00856664aec4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:14:57,736 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:57,736 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/A, priority=13, startTime=1733620497236; duration=0sec 2024-12-08T01:14:57,736 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:14:57,736 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:A 2024-12-08T01:14:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T01:14:57,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742041_1217 (size=22561) 2024-12-08T01:14:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733620497754, max timestamp=1733620497769Num uniques 1068; 2024-12-08T01:14:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733620497754, max timestamp=1733620497769Num uniques 1068; 2024-12-08T01:14:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T01:14:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T01:14:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733620497754, max timestamp=1733620497769Num uniques 1009; 2024-12-08T01:14:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-08T01:14:57,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733620497810, max timestamp=1733620497822Num uniques 1027; 2024-12-08T01:14:57,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733620497810, max timestamp=1733620497822Num uniques 1027; 2024-12-08T01:14:57,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T01:14:57,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T01:14:57,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733620497810, max timestamp=1733620497822Num uniques 1027; 2024-12-08T01:14:57,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T01:14:57,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733620497855, max timestamp=1733620497898Num uniques 1086; 2024-12-08T01:14:57,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-08T01:14:57,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733620497855, max timestamp=1733620497898Num uniques 1086; 2024-12-08T01:14:57,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-08T01:14:57,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733620497855, max timestamp=1733620497898Num uniques 1086; 2024-12-08T01:14:57,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-08T01:14:57,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733620497928, max timestamp=1733620497949Num uniques 1145; 2024-12-08T01:14:57,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733620497928, max timestamp=1733620497949Num uniques 1145; 2024-12-08T01:14:57,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733620497928, max timestamp=1733620497949Num uniques 1145; 2024-12-08T01:14:57,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-08T01:14:57,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-08T01:14:57,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-08T01:14:58,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620558001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620558010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620558122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620558122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,151 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=576, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5e58cdef74d644c39b0953c4cd87a94f 2024-12-08T01:14:58,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/26c38dfdf3b042bba3823ae29f6d3d1b is 50, key is test_row_0/B:col10/1733620495540/Put/seqid=0 2024-12-08T01:14:58,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742042_1218 (size=9857) 2024-12-08T01:14:58,171 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-12-08T01:14:58,171 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:58,171 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,172 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:58,173 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-12-08T01:14:58,173 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:58,173 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,173 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:58,176 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-12-08T01:14:58,176 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:58,176 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,176 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:58,179 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-12-08T01:14:58,179 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:58,179 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,179 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:58,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T01:14:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620558333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620558333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,564 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/26c38dfdf3b042bba3823ae29f6d3d1b 2024-12-08T01:14:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/ff23615c2e5148d59f3f259dbf05c972 is 50, key is test_row_0/C:col10/1733620495540/Put/seqid=0 2024-12-08T01:14:58,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742043_1219 (size=9857) 2024-12-08T01:14:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40718 deadline: 1733620558612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,613 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 56865e84e13eaff78d5439ba323ab5f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:14:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:14:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40740 deadline: 1733620558642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40698 deadline: 1733620558642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 2024-12-08T01:14:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:14:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:14:58,982 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/ff23615c2e5148d59f3f259dbf05c972 2024-12-08T01:14:58,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5e58cdef74d644c39b0953c4cd87a94f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f 2024-12-08T01:14:58,998 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f, entries=100, sequenceid=576, filesize=22.0 K 2024-12-08T01:14:58,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/26c38dfdf3b042bba3823ae29f6d3d1b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/26c38dfdf3b042bba3823ae29f6d3d1b 2024-12-08T01:14:59,004 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/26c38dfdf3b042bba3823ae29f6d3d1b, entries=100, sequenceid=576, filesize=9.6 K 2024-12-08T01:14:59,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/ff23615c2e5148d59f3f259dbf05c972 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/ff23615c2e5148d59f3f259dbf05c972 2024-12-08T01:14:59,008 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/ff23615c2e5148d59f3f259dbf05c972, entries=100, sequenceid=576, filesize=9.6 K 2024-12-08T01:14:59,009 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for 56865e84e13eaff78d5439ba323ab5f2 in 1743ms, sequenceid=576, compaction requested=false 2024-12-08T01:14:59,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:14:59,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:14:59,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-08T01:14:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-08T01:14:59,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-08T01:14:59,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8960 sec 2024-12-08T01:14:59,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9000 sec 2024-12-08T01:14:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:59,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=167.72 KB heapSize=472.55 KB 2024-12-08T01:14:59,157 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-12-08T01:14:59,158 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-12-08T01:14:59,158 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:59,158 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:14:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:14:59,158 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:59,158 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:59,158 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:59,158 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:14:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:14:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:59,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:14:59,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:59,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:14:59,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:14:59,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120843dab84604c94ffa8bd25963256034c7_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_0/A:col10/1733620498000/Put/seqid=0 2024-12-08T01:14:59,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742044_1220 (size=12454) 2024-12-08T01:14:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T01:14:59,272 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:14:59,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:14:59,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120843dab84604c94ffa8bd25963256034c7_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120843dab84604c94ffa8bd25963256034c7_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:14:59,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e7eaaa6779a473996216348353fcb26, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:14:59,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e7eaaa6779a473996216348353fcb26 is 175, key is test_row_0/A:col10/1733620498000/Put/seqid=0 2024-12-08T01:14:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742045_1221 (size=31255) 2024-12-08T01:14:59,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=686, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e7eaaa6779a473996216348353fcb26 2024-12-08T01:14:59,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/8c662c64932b48e2828f5f21c162b86a is 50, key is test_row_0/B:col10/1733620498000/Put/seqid=0 2024-12-08T01:14:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742046_1222 (size=12301) 2024-12-08T01:15:00,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/8c662c64932b48e2828f5f21c162b86a 2024-12-08T01:15:00,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/b01dbc87739e44828ce5b89acd7e9e30 is 50, key is test_row_0/C:col10/1733620498000/Put/seqid=0 2024-12-08T01:15:00,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742047_1223 (size=12301) 2024-12-08T01:15:00,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/b01dbc87739e44828ce5b89acd7e9e30 2024-12-08T01:15:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/6e7eaaa6779a473996216348353fcb26 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26 2024-12-08T01:15:00,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26, entries=150, sequenceid=686, filesize=30.5 K 2024-12-08T01:15:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/8c662c64932b48e2828f5f21c162b86a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/8c662c64932b48e2828f5f21c162b86a 2024-12-08T01:15:00,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/8c662c64932b48e2828f5f21c162b86a, entries=150, sequenceid=686, filesize=12.0 K 2024-12-08T01:15:00,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/b01dbc87739e44828ce5b89acd7e9e30 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b01dbc87739e44828ce5b89acd7e9e30 2024-12-08T01:15:00,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b01dbc87739e44828ce5b89acd7e9e30, entries=150, sequenceid=686, filesize=12.0 K 2024-12-08T01:15:00,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~472.50 KB/483840, currentSize=0 B/0 for 56865e84e13eaff78d5439ba323ab5f2 in 1688ms, sequenceid=686, compaction requested=true 2024-12-08T01:15:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:15:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:00,845 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:00,845 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:00,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:00,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56865e84e13eaff78d5439ba323ab5f2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:00,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:00,846 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:00,846 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:00,846 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/A is initiating minor compaction (all files) 2024-12-08T01:15:00,846 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/B is initiating minor compaction (all files) 2024-12-08T01:15:00,847 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/A in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:00,847 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/B in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:00,847 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/f29d05145ec24fc5beaab68d16131f83, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/26c38dfdf3b042bba3823ae29f6d3d1b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/8c662c64932b48e2828f5f21c162b86a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=34.3 K 2024-12-08T01:15:00,847 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/7c17badf13374154ad6e00856664aec4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=83.7 K 2024-12-08T01:15:00,847 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:00,847 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/7c17badf13374154ad6e00856664aec4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26] 2024-12-08T01:15:00,847 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f29d05145ec24fc5beaab68d16131f83, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1733620494426 2024-12-08T01:15:00,848 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c17badf13374154ad6e00856664aec4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1733620494426 2024-12-08T01:15:00,848 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c38dfdf3b042bba3823ae29f6d3d1b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733620495535 2024-12-08T01:15:00,848 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e58cdef74d644c39b0953c4cd87a94f, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733620495535 2024-12-08T01:15:00,848 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c662c64932b48e2828f5f21c162b86a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=686, earliestPutTs=1733620497999 2024-12-08T01:15:00,848 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7eaaa6779a473996216348353fcb26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=686, earliestPutTs=1733620497999 2024-12-08T01:15:00,889 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:15:00,890 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#B#compaction#186 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:00,891 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/e66320f6d5c34c198ba94b84e88bbfd6 is 50, key is test_row_0/B:col10/1733620498000/Put/seqid=0 2024-12-08T01:15:00,893 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208c40635c45bc246faa08b0104b9b378f0_56865e84e13eaff78d5439ba323ab5f2 store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:15:00,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742048_1224 (size=13051) 2024-12-08T01:15:00,923 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/e66320f6d5c34c198ba94b84e88bbfd6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e66320f6d5c34c198ba94b84e88bbfd6 2024-12-08T01:15:00,929 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/B of 56865e84e13eaff78d5439ba323ab5f2 into e66320f6d5c34c198ba94b84e88bbfd6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:00,929 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:15:00,930 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/B, priority=13, startTime=1733620500845; duration=0sec 2024-12-08T01:15:00,930 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:00,930 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:B 2024-12-08T01:15:00,930 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:00,931 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:00,931 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 56865e84e13eaff78d5439ba323ab5f2/C is initiating minor compaction (all files) 2024-12-08T01:15:00,931 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56865e84e13eaff78d5439ba323ab5f2/C in TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:00,932 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/a9d2e0d0d2a7475d9aa32539529e31e5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/ff23615c2e5148d59f3f259dbf05c972, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b01dbc87739e44828ce5b89acd7e9e30] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp, totalSize=34.3 K 2024-12-08T01:15:00,932 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a9d2e0d0d2a7475d9aa32539529e31e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1733620494426 2024-12-08T01:15:00,933 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting ff23615c2e5148d59f3f259dbf05c972, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733620495535 2024-12-08T01:15:00,933 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b01dbc87739e44828ce5b89acd7e9e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=686, earliestPutTs=1733620497999 2024-12-08T01:15:00,945 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208c40635c45bc246faa08b0104b9b378f0_56865e84e13eaff78d5439ba323ab5f2, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:15:00,945 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c40635c45bc246faa08b0104b9b378f0_56865e84e13eaff78d5439ba323ab5f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:15:00,988 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#C#compaction#188 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:00,988 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/3504c89710354fe1aec7202534d838e2 is 50, key is test_row_0/C:col10/1733620498000/Put/seqid=0 2024-12-08T01:15:00,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742049_1225 (size=4469) 2024-12-08T01:15:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742050_1226 (size=13051) 2024-12-08T01:15:01,397 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56865e84e13eaff78d5439ba323ab5f2#A#compaction#187 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:01,398 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/37af470a105b4a94a69e6c006a4d233d is 175, key is test_row_0/A:col10/1733620498000/Put/seqid=0 2024-12-08T01:15:01,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742051_1227 (size=32005) 2024-12-08T01:15:01,407 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/3504c89710354fe1aec7202534d838e2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3504c89710354fe1aec7202534d838e2 2024-12-08T01:15:01,413 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/C of 56865e84e13eaff78d5439ba323ab5f2 into 3504c89710354fe1aec7202534d838e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:01,413 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:15:01,413 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/C, priority=13, startTime=1733620500846; duration=0sec 2024-12-08T01:15:01,413 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:01,413 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:C 2024-12-08T01:15:01,814 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/37af470a105b4a94a69e6c006a4d233d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/37af470a105b4a94a69e6c006a4d233d 2024-12-08T01:15:01,819 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56865e84e13eaff78d5439ba323ab5f2/A of 56865e84e13eaff78d5439ba323ab5f2 into 37af470a105b4a94a69e6c006a4d233d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:01,819 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56865e84e13eaff78d5439ba323ab5f2: 2024-12-08T01:15:01,819 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., storeName=56865e84e13eaff78d5439ba323ab5f2/A, priority=13, startTime=1733620500845; duration=0sec 2024-12-08T01:15:01,819 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:01,819 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56865e84e13eaff78d5439ba323ab5f2:A 2024-12-08T01:15:02,637 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-12-08T01:15:02,637 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:02,637 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:02,638 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:02,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:06,407 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-12-08T01:15:06,407 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:06,408 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:06,408 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:06,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:06,447 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-12-08T01:15:06,447 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:06,447 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:06,448 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:06,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T01:15:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3728 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3683 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1603 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4807 rows 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1616 2024-12-08T01:15:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4837 rows 2024-12-08T01:15:06,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:15:06,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:15:06,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:06,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:06,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:06,451 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T01:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T01:15:06,457 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620506456"}]},"ts":"1733620506456"} 2024-12-08T01:15:06,458 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T01:15:06,458 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T01:15:06,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:15:06,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, UNASSIGN}] 2024-12-08T01:15:06,460 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, UNASSIGN 2024-12-08T01:15:06,461 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:06,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, UNASSIGN because future has completed 2024-12-08T01:15:06,463 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:15:06,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630}] 2024-12-08T01:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T01:15:06,616 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:06,616 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:15:06,617 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 56865e84e13eaff78d5439ba323ab5f2, disabling compactions & flushes 2024-12-08T01:15:06,617 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:06,617 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:06,617 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. after waiting 0 ms 2024-12-08T01:15:06,617 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:06,617 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 56865e84e13eaff78d5439ba323ab5f2 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T01:15:06,618 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=A 2024-12-08T01:15:06,618 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:06,618 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=B 2024-12-08T01:15:06,618 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:06,618 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56865e84e13eaff78d5439ba323ab5f2, store=C 2024-12-08T01:15:06,618 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:06,629 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087812fc48cac3445a97b7899858c8cfa6_56865e84e13eaff78d5439ba323ab5f2 is 50, key is test_row_1/A:col10/1733620506404/Put/seqid=0 2024-12-08T01:15:06,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742052_1228 (size=9914) 2024-12-08T01:15:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T01:15:07,034 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:07,043 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087812fc48cac3445a97b7899858c8cfa6_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087812fc48cac3445a97b7899858c8cfa6_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:07,044 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5867275b499d41b987673b10029e774f, store: [table=TestAcidGuarantees family=A region=56865e84e13eaff78d5439ba323ab5f2] 2024-12-08T01:15:07,045 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5867275b499d41b987673b10029e774f is 175, key is test_row_1/A:col10/1733620506404/Put/seqid=0 2024-12-08T01:15:07,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742053_1229 (size=22561) 2024-12-08T01:15:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T01:15:07,450 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=695, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5867275b499d41b987673b10029e774f 2024-12-08T01:15:07,464 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/d459fd614434423da8a7af5ae4b2dc26 is 50, key is test_row_1/B:col10/1733620506404/Put/seqid=0 2024-12-08T01:15:07,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742054_1230 (size=9857) 2024-12-08T01:15:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T01:15:07,870 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/d459fd614434423da8a7af5ae4b2dc26 2024-12-08T01:15:07,887 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/df97c7a356ca4000bc7333eff4d9c987 is 50, key is test_row_1/C:col10/1733620506404/Put/seqid=0 2024-12-08T01:15:07,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742055_1231 (size=9857) 2024-12-08T01:15:08,292 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/df97c7a356ca4000bc7333eff4d9c987 2024-12-08T01:15:08,303 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/A/5867275b499d41b987673b10029e774f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5867275b499d41b987673b10029e774f 2024-12-08T01:15:08,310 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5867275b499d41b987673b10029e774f, entries=100, sequenceid=695, filesize=22.0 K 2024-12-08T01:15:08,312 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/B/d459fd614434423da8a7af5ae4b2dc26 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/d459fd614434423da8a7af5ae4b2dc26 2024-12-08T01:15:08,318 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/d459fd614434423da8a7af5ae4b2dc26, entries=100, sequenceid=695, filesize=9.6 K 2024-12-08T01:15:08,319 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/.tmp/C/df97c7a356ca4000bc7333eff4d9c987 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/df97c7a356ca4000bc7333eff4d9c987 2024-12-08T01:15:08,325 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/df97c7a356ca4000bc7333eff4d9c987, entries=100, sequenceid=695, filesize=9.6 K 2024-12-08T01:15:08,326 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 56865e84e13eaff78d5439ba323ab5f2 in 1709ms, sequenceid=695, compaction requested=false 2024-12-08T01:15:08,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/e66ad5e686ff4e60838dccea72a9051b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/091fedcfecd044329073e3a75d1764b4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/98d88d760d3649758432def0d9a25c4d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/d9b6c1b010c2463ea1e31b93a7d17e27, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/7c17badf13374154ad6e00856664aec4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26] to archive 2024-12-08T01:15:08,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:08,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e6d4f8abb2641c788ce1c62fbb17d1d 2024-12-08T01:15:08,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/4abfcae108b74b0bb4eefcde6d0b914e 2024-12-08T01:15:08,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/8ac9be93943c442b9b17fe69eb75e4b5 2024-12-08T01:15:08,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/15d5d406283e4f48a1b2512da65c6b76 2024-12-08T01:15:08,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/e66ad5e686ff4e60838dccea72a9051b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/e66ad5e686ff4e60838dccea72a9051b 2024-12-08T01:15:08,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/3549e290f8774ad29f1725572deb9353 2024-12-08T01:15:08,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f2da1a32165e465ab1e5076b1bc77de9 2024-12-08T01:15:08,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/091fedcfecd044329073e3a75d1764b4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/091fedcfecd044329073e3a75d1764b4 2024-12-08T01:15:08,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/501edd23aa4e4eb1bac9cd9ce46c125d 2024-12-08T01:15:08,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f341848d248440aba641f82cb1e6fed4 2024-12-08T01:15:08,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/c6e34c3ccc364f25b2a44916bbb5053f 2024-12-08T01:15:08,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/b617a37c866243e28fc4602c85546e7e 2024-12-08T01:15:08,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/98d88d760d3649758432def0d9a25c4d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/98d88d760d3649758432def0d9a25c4d 2024-12-08T01:15:08,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/14c218da13dc4cd7aaedbf36a300d741 2024-12-08T01:15:08,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/cf39242e052040f687efb3b9262484cc 2024-12-08T01:15:08,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/755a092ce8c942cea5acdb338c0f0c4b 2024-12-08T01:15:08,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/d9b6c1b010c2463ea1e31b93a7d17e27 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/d9b6c1b010c2463ea1e31b93a7d17e27 2024-12-08T01:15:08,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/f12a52201fb34edbb9a05cc3504268f1 2024-12-08T01:15:08,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/9cc8f79720dd46cfa9f8764d61a55492 2024-12-08T01:15:08,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/7c17badf13374154ad6e00856664aec4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/7c17badf13374154ad6e00856664aec4 2024-12-08T01:15:08,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5e58cdef74d644c39b0953c4cd87a94f 2024-12-08T01:15:08,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/6e7eaaa6779a473996216348353fcb26 2024-12-08T01:15:08,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/1f0b4c5ea7f74dceba6b605d07d18491, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/62fdafcd37774e4b89b65b295994b4f8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3afe240033bf4b00bbab5e7d338be9c1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/28b72b9cd98d4c05bb058e652f36a160, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/632ac03afd7e42e7af2a3f71979a37c6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/c189089ebc2b46809e1d0a0d73eeb976, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/7550e33abb7e4195bf69683e806147d0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/b796c4d11ab4403aab71f72c5e569bea, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e07ee19af63d44689faf162d9d8fcf30, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/2470b87ffd854b0a81d607616aad3d7f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/366ccb5be2084fe588d00f542a130b7d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/dfb2d4b8765b46c2a4513d52f736dd2a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/456126ef7fc34b088c229a0149423a09, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/bd2a641e9678488d96ee5d3dbb57bcb0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/85ba503c1e1248afa73ef343e5f28aaa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3dc8a4c6f4474ac78cb3b85e27627609, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/74532796b1614206a2b538f847f5e266, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/f29d05145ec24fc5beaab68d16131f83, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/13a73955fd4c43059d0119fefec4c4ed, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/26c38dfdf3b042bba3823ae29f6d3d1b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/8c662c64932b48e2828f5f21c162b86a] to archive 2024-12-08T01:15:08,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:08,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/1f0b4c5ea7f74dceba6b605d07d18491 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/1f0b4c5ea7f74dceba6b605d07d18491 2024-12-08T01:15:08,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/62fdafcd37774e4b89b65b295994b4f8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/62fdafcd37774e4b89b65b295994b4f8 2024-12-08T01:15:08,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3afe240033bf4b00bbab5e7d338be9c1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3afe240033bf4b00bbab5e7d338be9c1 2024-12-08T01:15:08,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/28b72b9cd98d4c05bb058e652f36a160 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/28b72b9cd98d4c05bb058e652f36a160 2024-12-08T01:15:08,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/632ac03afd7e42e7af2a3f71979a37c6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/632ac03afd7e42e7af2a3f71979a37c6 2024-12-08T01:15:08,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/a86eadcd9c5e4f19b0b2db68e5a2dbaa 2024-12-08T01:15:08,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/c189089ebc2b46809e1d0a0d73eeb976 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/c189089ebc2b46809e1d0a0d73eeb976 2024-12-08T01:15:08,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/7550e33abb7e4195bf69683e806147d0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/7550e33abb7e4195bf69683e806147d0 2024-12-08T01:15:08,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/b796c4d11ab4403aab71f72c5e569bea to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/b796c4d11ab4403aab71f72c5e569bea 2024-12-08T01:15:08,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e07ee19af63d44689faf162d9d8fcf30 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e07ee19af63d44689faf162d9d8fcf30 2024-12-08T01:15:08,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/2470b87ffd854b0a81d607616aad3d7f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/2470b87ffd854b0a81d607616aad3d7f 2024-12-08T01:15:08,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/366ccb5be2084fe588d00f542a130b7d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/366ccb5be2084fe588d00f542a130b7d 2024-12-08T01:15:08,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/dfb2d4b8765b46c2a4513d52f736dd2a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/dfb2d4b8765b46c2a4513d52f736dd2a 2024-12-08T01:15:08,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/456126ef7fc34b088c229a0149423a09 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/456126ef7fc34b088c229a0149423a09 2024-12-08T01:15:08,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/bd2a641e9678488d96ee5d3dbb57bcb0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/bd2a641e9678488d96ee5d3dbb57bcb0 2024-12-08T01:15:08,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/85ba503c1e1248afa73ef343e5f28aaa to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/85ba503c1e1248afa73ef343e5f28aaa 2024-12-08T01:15:08,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3dc8a4c6f4474ac78cb3b85e27627609 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/3dc8a4c6f4474ac78cb3b85e27627609 2024-12-08T01:15:08,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/74532796b1614206a2b538f847f5e266 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/74532796b1614206a2b538f847f5e266 2024-12-08T01:15:08,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/f29d05145ec24fc5beaab68d16131f83 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/f29d05145ec24fc5beaab68d16131f83 2024-12-08T01:15:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/13a73955fd4c43059d0119fefec4c4ed to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/13a73955fd4c43059d0119fefec4c4ed 2024-12-08T01:15:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/26c38dfdf3b042bba3823ae29f6d3d1b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/26c38dfdf3b042bba3823ae29f6d3d1b 2024-12-08T01:15:08,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/8c662c64932b48e2828f5f21c162b86a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/8c662c64932b48e2828f5f21c162b86a 2024-12-08T01:15:08,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/47b01471ff7943ba95ed3d83f1990c43, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/093ea805583a4ea0b741a8af6c808eac, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/f72e9910f03e4be2a1ae113584393fa2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/0efe96aa003e4308933665a5d393fe4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3e114f4167344f0f9539bf434a736caf, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/159f1984d4774804bc8f094b24e96ccb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/482af29b24c14c97ba645c8cd6c5d524, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/867c3c41bf624efcaa705e7ca6b9fcb7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/e7887cbd64f14e4d88558c1b6815d161, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/043340db782048c886ad0cb72173bff8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/d764d41b07924d6bbef503a67fa8d13d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/9867589da4b34bd78ade5a3e5036d252, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/8a975bdef8d54f26a8cfdc60160fb154, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/c46fd6b6fbe349bb8ec1e2b41b757a28, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/65ff5d023e3f413aa5d00cddc51be467, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b708a78ef90c499f9959bd44631a3f98, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/450992920282413a9601188d1539caf2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/899541a7276d404296c497bb668a5fd4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/a9d2e0d0d2a7475d9aa32539529e31e5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/6f5bf46875724e07ab48c804e083d134, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/ff23615c2e5148d59f3f259dbf05c972, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b01dbc87739e44828ce5b89acd7e9e30] to archive 2024-12-08T01:15:08,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:08,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/47b01471ff7943ba95ed3d83f1990c43 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/47b01471ff7943ba95ed3d83f1990c43 2024-12-08T01:15:08,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/093ea805583a4ea0b741a8af6c808eac to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/093ea805583a4ea0b741a8af6c808eac 2024-12-08T01:15:08,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/f72e9910f03e4be2a1ae113584393fa2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/f72e9910f03e4be2a1ae113584393fa2 2024-12-08T01:15:08,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/0efe96aa003e4308933665a5d393fe4e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/0efe96aa003e4308933665a5d393fe4e 2024-12-08T01:15:08,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3e114f4167344f0f9539bf434a736caf to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3e114f4167344f0f9539bf434a736caf 2024-12-08T01:15:08,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/159f1984d4774804bc8f094b24e96ccb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/159f1984d4774804bc8f094b24e96ccb 2024-12-08T01:15:08,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/482af29b24c14c97ba645c8cd6c5d524 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/482af29b24c14c97ba645c8cd6c5d524 2024-12-08T01:15:08,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/867c3c41bf624efcaa705e7ca6b9fcb7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/867c3c41bf624efcaa705e7ca6b9fcb7 2024-12-08T01:15:08,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/e7887cbd64f14e4d88558c1b6815d161 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/e7887cbd64f14e4d88558c1b6815d161 2024-12-08T01:15:08,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/043340db782048c886ad0cb72173bff8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/043340db782048c886ad0cb72173bff8 2024-12-08T01:15:08,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/d764d41b07924d6bbef503a67fa8d13d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/d764d41b07924d6bbef503a67fa8d13d 2024-12-08T01:15:08,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/9867589da4b34bd78ade5a3e5036d252 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/9867589da4b34bd78ade5a3e5036d252 2024-12-08T01:15:08,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/8a975bdef8d54f26a8cfdc60160fb154 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/8a975bdef8d54f26a8cfdc60160fb154 2024-12-08T01:15:08,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/c46fd6b6fbe349bb8ec1e2b41b757a28 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/c46fd6b6fbe349bb8ec1e2b41b757a28 2024-12-08T01:15:08,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/65ff5d023e3f413aa5d00cddc51be467 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/65ff5d023e3f413aa5d00cddc51be467 2024-12-08T01:15:08,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b708a78ef90c499f9959bd44631a3f98 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b708a78ef90c499f9959bd44631a3f98 2024-12-08T01:15:08,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/450992920282413a9601188d1539caf2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/450992920282413a9601188d1539caf2 2024-12-08T01:15:08,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/899541a7276d404296c497bb668a5fd4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/899541a7276d404296c497bb668a5fd4 2024-12-08T01:15:08,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/a9d2e0d0d2a7475d9aa32539529e31e5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/a9d2e0d0d2a7475d9aa32539529e31e5 2024-12-08T01:15:08,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/6f5bf46875724e07ab48c804e083d134 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/6f5bf46875724e07ab48c804e083d134 2024-12-08T01:15:08,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/ff23615c2e5148d59f3f259dbf05c972 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/ff23615c2e5148d59f3f259dbf05c972 2024-12-08T01:15:08,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b01dbc87739e44828ce5b89acd7e9e30 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/b01dbc87739e44828ce5b89acd7e9e30 2024-12-08T01:15:08,393 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/recovered.edits/698.seqid, newMaxSeqId=698, maxSeqId=4 2024-12-08T01:15:08,393 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. 2024-12-08T01:15:08,394 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 56865e84e13eaff78d5439ba323ab5f2: Waiting for close lock at 1733620506617Running coprocessor pre-close hooks at 1733620506617Disabling compacts and flushes for region at 1733620506617Disabling writes for close at 1733620506617Obtaining lock to block concurrent updates at 1733620506617Preparing flush snapshotting stores in 56865e84e13eaff78d5439ba323ab5f2 at 1733620506617Finished memstore snapshotting TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733620506618 (+1 ms)Flushing stores of TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2. at 1733620506620 (+2 ms)Flushing 56865e84e13eaff78d5439ba323ab5f2/A: creating writer at 1733620506620Flushing mob file 56865e84e13eaff78d5439ba323ab5f2/A: appending metadata at 1733620506629 (+9 ms)Flushing mob file 56865e84e13eaff78d5439ba323ab5f2/A: closing flushed file at 1733620506629Flushing 56865e84e13eaff78d5439ba323ab5f2/A: appending metadata at 1733620507044 (+415 ms)Flushing 56865e84e13eaff78d5439ba323ab5f2/A: closing flushed file at 1733620507044Flushing 56865e84e13eaff78d5439ba323ab5f2/B: creating writer at 1733620507460 (+416 ms)Flushing 56865e84e13eaff78d5439ba323ab5f2/B: appending metadata at 1733620507463 (+3 ms)Flushing 56865e84e13eaff78d5439ba323ab5f2/B: closing flushed file at 1733620507463Flushing 56865e84e13eaff78d5439ba323ab5f2/C: creating writer at 1733620507883 (+420 ms)Flushing 56865e84e13eaff78d5439ba323ab5f2/C: appending metadata at 1733620507886 (+3 ms)Flushing 56865e84e13eaff78d5439ba323ab5f2/C: closing flushed file at 1733620507886Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7204b10b: reopening flushed file at 1733620508302 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c64ae0e: reopening flushed file at 1733620508310 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b022b3: reopening flushed file at 1733620508318 (+8 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 56865e84e13eaff78d5439ba323ab5f2 in 1709ms, sequenceid=695, compaction requested=false at 1733620508326 (+8 ms)Writing region close event to WAL at 1733620508390 (+64 ms)Running coprocessor post-close hooks at 1733620508393 (+3 ms)Closed at 1733620508393 2024-12-08T01:15:08,395 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=56865e84e13eaff78d5439ba323ab5f2, regionState=CLOSED 2024-12-08T01:15:08,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:15:08,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-08T01:15:08,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 56865e84e13eaff78d5439ba323ab5f2, server=96982154c400,44843,1733620436630 in 1.9340 sec 2024-12-08T01:15:08,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-08T01:15:08,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56865e84e13eaff78d5439ba323ab5f2, UNASSIGN in 1.9390 sec 2024-12-08T01:15:08,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-08T01:15:08,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9420 sec 2024-12-08T01:15:08,402 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620508402"}]},"ts":"1733620508402"} 2024-12-08T01:15:08,404 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T01:15:08,404 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T01:15:08,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9530 sec 2024-12-08T01:15:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T01:15:08,603 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:08,604 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T01:15:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,608 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T01:15:08,610 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,614 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/recovered.edits] 2024-12-08T01:15:08,621 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/37af470a105b4a94a69e6c006a4d233d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/37af470a105b4a94a69e6c006a4d233d 2024-12-08T01:15:08,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5867275b499d41b987673b10029e774f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/A/5867275b499d41b987673b10029e774f 2024-12-08T01:15:08,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/d459fd614434423da8a7af5ae4b2dc26 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/d459fd614434423da8a7af5ae4b2dc26 2024-12-08T01:15:08,627 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e66320f6d5c34c198ba94b84e88bbfd6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/B/e66320f6d5c34c198ba94b84e88bbfd6 2024-12-08T01:15:08,631 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3504c89710354fe1aec7202534d838e2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/3504c89710354fe1aec7202534d838e2 2024-12-08T01:15:08,633 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/df97c7a356ca4000bc7333eff4d9c987 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/C/df97c7a356ca4000bc7333eff4d9c987 2024-12-08T01:15:08,637 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/recovered.edits/698.seqid to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2/recovered.edits/698.seqid 2024-12-08T01:15:08,637 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,637 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T01:15:08,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T01:15:08,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T01:15:08,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080e35c22367534ffab183126163d36ab0_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080e35c22367534ffab183126163d36ab0_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208169ed9b97cc84e1c83fb4376e6e3e8b1_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208169ed9b97cc84e1c83fb4376e6e3e8b1_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120822a65cc032c24cb9ab6eb850266168e8_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120822a65cc032c24cb9ab6eb850266168e8_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,644 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120833608170aacf4aa092be05bf5773dec0_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120833608170aacf4aa092be05bf5773dec0_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,645 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083996ae5bab65475da0170140f3e169d9_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083996ae5bab65475da0170140f3e169d9_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,646 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120843dab84604c94ffa8bd25963256034c7_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120843dab84604c94ffa8bd25963256034c7_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,647 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084c3fdb5c31274cb1859f6e3b11c83594_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084c3fdb5c31274cb1859f6e3b11c83594_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,648 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120850d5ad6eb0c7462e88bfcf7457e7f55b_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120850d5ad6eb0c7462e88bfcf7457e7f55b_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120851b954b65cae423a8724d9dbd9bf0b7b_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120851b954b65cae423a8724d9dbd9bf0b7b_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208632e41fa09f2407dad372b8c311c7ad1_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208632e41fa09f2407dad372b8c311c7ad1_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,651 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086a569a793edf486bb32eb25946f190d2_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086a569a793edf486bb32eb25946f190d2_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,653 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120871b11870429a4dd2bcbdcd9641a9e8da_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120871b11870429a4dd2bcbdcd9641a9e8da_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,654 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087812fc48cac3445a97b7899858c8cfa6_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087812fc48cac3445a97b7899858c8cfa6_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a30a77e22fd341ada8d561f846ba2c43_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a30a77e22fd341ada8d561f846ba2c43_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,656 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a8fb2af17cd74a8ab7e5cca7e52e8039_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a8fb2af17cd74a8ab7e5cca7e52e8039_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c36c3d2db2de4fa498e5617d63512d7b_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c36c3d2db2de4fa498e5617d63512d7b_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,658 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e92b5e58716540ec800ee2ba23d0ec66_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e92b5e58716540ec800ee2ba23d0ec66_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe70492f2f7347b886132d2541b23c1a_56865e84e13eaff78d5439ba323ab5f2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe70492f2f7347b886132d2541b23c1a_56865e84e13eaff78d5439ba323ab5f2 2024-12-08T01:15:08,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T01:15:08,662 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,664 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T01:15:08,666 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T01:15:08,668 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,668 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T01:15:08,668 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733620508668"}]},"ts":"9223372036854775807"} 2024-12-08T01:15:08,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T01:15:08,670 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 56865e84e13eaff78d5439ba323ab5f2, NAME => 'TestAcidGuarantees,,1733620475004.56865e84e13eaff78d5439ba323ab5f2.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T01:15:08,670 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T01:15:08,670 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733620508670"}]},"ts":"9223372036854775807"} 2024-12-08T01:15:08,671 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T01:15:08,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-12-08T01:15:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T01:15:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T01:15:08,712 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:08,725 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-207628044_22 at /127.0.0.1:39178 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 326) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9427 (was 9615) 2024-12-08T01:15:08,735 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=9427 2024-12-08T01:15:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:15:08,736 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:15:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:08,739 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:15:08,739 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:08,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-08T01:15:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T01:15:08,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:15:08,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742056_1232 (size=1061) 2024-12-08T01:15:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T01:15:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T01:15:09,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:15:09,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742057_1233 (size=53) 2024-12-08T01:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T01:15:09,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:15:09,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 836035c90977e14b643a4b4ed6128d49, disabling compactions & flushes 2024-12-08T01:15:09,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. after waiting 0 ms 2024-12-08T01:15:09,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 836035c90977e14b643a4b4ed6128d49: Waiting for close lock at 1733620509566Disabling compacts and flushes for region at 1733620509566Disabling writes for close at 1733620509566Writing region close event to WAL at 1733620509566Closed at 1733620509566 2024-12-08T01:15:09,569 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:15:09,570 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733620509569"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620509569"}]},"ts":"1733620509569"} 2024-12-08T01:15:09,574 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:15:09,576 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:15:09,576 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620509576"}]},"ts":"1733620509576"} 2024-12-08T01:15:09,578 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T01:15:09,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, ASSIGN}] 2024-12-08T01:15:09,580 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, ASSIGN 2024-12-08T01:15:09,581 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, ASSIGN; state=OFFLINE, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=false 2024-12-08T01:15:09,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=836035c90977e14b643a4b4ed6128d49, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:09,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, ASSIGN because future has completed 2024-12-08T01:15:09,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630}] 2024-12-08T01:15:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T01:15:09,899 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,899 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:15:09,900 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,900 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:15:09,900 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,900 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,902 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,904 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:09,904 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 836035c90977e14b643a4b4ed6128d49 columnFamilyName A 2024-12-08T01:15:09,904 DEBUG [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:09,905 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.HStore(327): Store=836035c90977e14b643a4b4ed6128d49/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:09,905 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,906 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:09,906 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 836035c90977e14b643a4b4ed6128d49 columnFamilyName B 2024-12-08T01:15:09,906 DEBUG [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:09,906 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.HStore(327): Store=836035c90977e14b643a4b4ed6128d49/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:09,907 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,907 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:09,907 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 836035c90977e14b643a4b4ed6128d49 columnFamilyName C 2024-12-08T01:15:09,908 DEBUG [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:09,908 INFO [StoreOpener-836035c90977e14b643a4b4ed6128d49-1 {}] regionserver.HStore(327): Store=836035c90977e14b643a4b4ed6128d49/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:09,908 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,908 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,909 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,909 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,910 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,910 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,910 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:15:09,911 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,913 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:15:09,913 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 836035c90977e14b643a4b4ed6128d49; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60583293, jitterRate=-0.09723858535289764}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:15:09,913 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:09,914 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 836035c90977e14b643a4b4ed6128d49: Running coprocessor pre-open hook at 1733620509900Writing region info on filesystem at 1733620509900Initializing all the Stores at 1733620509902 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620509902Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620509902Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620509902Cleaning up temporary data from old regions at 1733620509910 (+8 ms)Running coprocessor post-open hooks at 1733620509913 (+3 ms)Region opened successfully at 1733620509914 (+1 ms) 2024-12-08T01:15:09,914 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., pid=65, masterSystemTime=1733620509892 2024-12-08T01:15:09,916 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,916 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:09,916 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=836035c90977e14b643a4b4ed6128d49, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:09,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:15:09,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-08T01:15:09,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 in 181 msec 2024-12-08T01:15:09,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-08T01:15:09,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, ASSIGN in 342 msec 2024-12-08T01:15:09,923 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:15:09,923 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620509923"}]},"ts":"1733620509923"} 2024-12-08T01:15:09,924 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T01:15:09,925 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:15:09,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-12-08T01:15:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T01:15:10,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:10,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e941646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5275516e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,893 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:15:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:15:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@701df2f4 2024-12-08T01:15:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:15:10,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:15:10,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,902 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5632c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c04a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,907 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d57cdcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a294c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683edcca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1569f785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39857722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,916 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3ce829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df09263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4654eddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2157be9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,922 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64749ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,923 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7a1f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,925 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ace379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,926 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d89893c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,927 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff8914f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,929 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb7356d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee0da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:10,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:10,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331f7bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:10,934 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574ad43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:10,935 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,935 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,935 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,936 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,937 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,938 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,938 DEBUG [Thread-1104 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,938 DEBUG [Thread-1104 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,938 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-08T01:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,939 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,940 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,942 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,945 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:10,945 DEBUG [Thread-1095 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,944 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:10,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:10,945 DEBUG [Thread-1095 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,947 DEBUG [Thread-1099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:10,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:10,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:15:10,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:10,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:10,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:10,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:10,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:10,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:10,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:15:10,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/89c7789635e341d991009829bbbe5cf3 is 50, key is test_row_0/A:col10/1733620510947/Put/seqid=0 2024-12-08T01:15:10,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620510961, max timestamp=1733620510978Num uniques 859; 2024-12-08T01:15:10,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620510961, max timestamp=1733620510978Num uniques 859; 2024-12-08T01:15:10,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:15:10,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:15:10,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620510961, max timestamp=1733620510978Num uniques 859; 2024-12-08T01:15:10,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:15:10,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742058_1234 (size=12001) 2024-12-08T01:15:10,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/89c7789635e341d991009829bbbe5cf3 2024-12-08T01:15:11,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620570996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620570996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620570997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620570999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620571002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/7da04854b15b4d5faa86bb0f92ed82d4 is 50, key is test_row_0/B:col10/1733620510947/Put/seqid=0 2024-12-08T01:15:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742059_1235 (size=12001) 2024-12-08T01:15:11,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620571112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620571112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620571112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620571112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620571112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620571322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620571322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620571322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620571323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620571323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/7da04854b15b4d5faa86bb0f92ed82d4 2024-12-08T01:15:11,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/92f92f32d2ab455e9110498f89d174de is 50, key is test_row_0/C:col10/1733620510947/Put/seqid=0 2024-12-08T01:15:11,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742060_1236 (size=12001) 2024-12-08T01:15:11,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/92f92f32d2ab455e9110498f89d174de 2024-12-08T01:15:11,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:11,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:11,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/89c7789635e341d991009829bbbe5cf3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/89c7789635e341d991009829bbbe5cf3 2024-12-08T01:15:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:11,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/89c7789635e341d991009829bbbe5cf3, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T01:15:11,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/7da04854b15b4d5faa86bb0f92ed82d4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/7da04854b15b4d5faa86bb0f92ed82d4 2024-12-08T01:15:11,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/7da04854b15b4d5faa86bb0f92ed82d4, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T01:15:11,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/92f92f32d2ab455e9110498f89d174de as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/92f92f32d2ab455e9110498f89d174de 2024-12-08T01:15:11,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/92f92f32d2ab455e9110498f89d174de, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T01:15:11,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 836035c90977e14b643a4b4ed6128d49 in 641ms, sequenceid=12, compaction requested=false 2024-12-08T01:15:11,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:11,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-08T01:15:11,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620510993, max timestamp=1733620511637Num uniques 1118; 2024-12-08T01:15:11,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-08T01:15:11,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733620510993, max timestamp=1733620511637Num uniques 1118; 2024-12-08T01:15:11,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-08T01:15:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T01:15:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:15:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:15:11,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/6f0ab344a2a64238932fbc628670aba1 is 50, key is test_row_0/A:col10/1733620511633/Put/seqid=0 2024-12-08T01:15:11,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742061_1237 (size=12001) 2024-12-08T01:15:11,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620571670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620571671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620571674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620571675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620571680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/6f0ab344a2a64238932fbc628670aba1 2024-12-08T01:15:11,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/243bd9db507f48238f13d0c036c54d8f is 50, key is test_row_0/B:col10/1733620511633/Put/seqid=0 2024-12-08T01:15:11,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:11,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:11,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742062_1238 (size=12001) 2024-12-08T01:15:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620571786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620571786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620571792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620571792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620571793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:11,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:11,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:11,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:11,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620572002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620572002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620572002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620572003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620572003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:12,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:12,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:12,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:12,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:12,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:12,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/243bd9db507f48238f13d0c036c54d8f 2024-12-08T01:15:12,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5a8da3afccf040cd8e49bf5866439a57 is 50, key is test_row_0/C:col10/1733620511633/Put/seqid=0 2024-12-08T01:15:12,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742063_1239 (size=12001) 2024-12-08T01:15:12,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5a8da3afccf040cd8e49bf5866439a57 2024-12-08T01:15:12,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/6f0ab344a2a64238932fbc628670aba1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/6f0ab344a2a64238932fbc628670aba1 2024-12-08T01:15:12,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/6f0ab344a2a64238932fbc628670aba1, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:15:12,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/243bd9db507f48238f13d0c036c54d8f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/243bd9db507f48238f13d0c036c54d8f 2024-12-08T01:15:12,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/243bd9db507f48238f13d0c036c54d8f, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:15:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5a8da3afccf040cd8e49bf5866439a57 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5a8da3afccf040cd8e49bf5866439a57 2024-12-08T01:15:12,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5a8da3afccf040cd8e49bf5866439a57, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:15:12,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~209.22 KB/214240, currentSize=107.34 KB/109920 for 836035c90977e14b643a4b4ed6128d49 in 538ms, sequenceid=50, compaction requested=false 2024-12-08T01:15:12,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:12,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T01:15:12,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:12,176 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T01:15:12,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:12,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:12,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/31a9147bc4d741d3a064c4b0c0d38053 is 50, key is test_row_0/A:col10/1733620511674/Put/seqid=0 2024-12-08T01:15:12,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742064_1240 (size=12001) 2024-12-08T01:15:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:12,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:12,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620572321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620572322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620572323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620572325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620572325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620572432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620572432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620572432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620572432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620572432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,592 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/31a9147bc4d741d3a064c4b0c0d38053 2024-12-08T01:15:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/4b93dda010e84e36942e700f59f5a62a is 50, key is test_row_0/B:col10/1733620511674/Put/seqid=0 2024-12-08T01:15:12,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742065_1241 (size=12001) 2024-12-08T01:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620572642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620572642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620572642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620572642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620572643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620572952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620572952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620572952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620572952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620572952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,011 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/4b93dda010e84e36942e700f59f5a62a 2024-12-08T01:15:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/46aeb8952c1e470eb443c076a00a842a is 50, key is test_row_0/C:col10/1733620511674/Put/seqid=0 2024-12-08T01:15:13,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742066_1242 (size=12001) 2024-12-08T01:15:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:13,444 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/46aeb8952c1e470eb443c076a00a842a 2024-12-08T01:15:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/31a9147bc4d741d3a064c4b0c0d38053 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/31a9147bc4d741d3a064c4b0c0d38053 2024-12-08T01:15:13,455 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/31a9147bc4d741d3a064c4b0c0d38053, entries=150, sequenceid=69, filesize=11.7 K 2024-12-08T01:15:13,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/4b93dda010e84e36942e700f59f5a62a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4b93dda010e84e36942e700f59f5a62a 2024-12-08T01:15:13,461 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4b93dda010e84e36942e700f59f5a62a, entries=150, sequenceid=69, filesize=11.7 K 2024-12-08T01:15:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/46aeb8952c1e470eb443c076a00a842a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/46aeb8952c1e470eb443c076a00a842a 2024-12-08T01:15:13,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620573463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,467 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/46aeb8952c1e470eb443c076a00a842a, entries=150, sequenceid=69, filesize=11.7 K 2024-12-08T01:15:13,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620573463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,468 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 836035c90977e14b643a4b4ed6128d49 in 1292ms, sequenceid=69, compaction requested=true 2024-12-08T01:15:13,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:13,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:13,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-08T01:15:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-08T01:15:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:13,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:15:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:13,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-08T01:15:13,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5240 sec 2024-12-08T01:15:13,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.5380 sec 2024-12-08T01:15:13,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/eb53ab1aeb6a4ce695702d98786633a8 is 50, key is test_row_0/A:col10/1733620512323/Put/seqid=0 2024-12-08T01:15:13,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742067_1243 (size=12001) 2024-12-08T01:15:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620573499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/eb53ab1aeb6a4ce695702d98786633a8 2024-12-08T01:15:13,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620573503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620573503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d01d0626459949cdbd377c21ba914c7a is 50, key is test_row_0/B:col10/1733620512323/Put/seqid=0 2024-12-08T01:15:13,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742068_1244 (size=12001) 2024-12-08T01:15:13,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d01d0626459949cdbd377c21ba914c7a 2024-12-08T01:15:13,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/4af707910eea47a7af1ed2ae892e3987 is 50, key is test_row_0/C:col10/1733620512323/Put/seqid=0 2024-12-08T01:15:13,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742069_1245 (size=12001) 2024-12-08T01:15:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620573612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620573612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620573612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620573822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620573822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620573823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:13,931 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:15:13,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/4af707910eea47a7af1ed2ae892e3987 2024-12-08T01:15:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/eb53ab1aeb6a4ce695702d98786633a8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/eb53ab1aeb6a4ce695702d98786633a8 2024-12-08T01:15:13,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/eb53ab1aeb6a4ce695702d98786633a8, entries=150, sequenceid=84, filesize=11.7 K 2024-12-08T01:15:13,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d01d0626459949cdbd377c21ba914c7a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d01d0626459949cdbd377c21ba914c7a 2024-12-08T01:15:14,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d01d0626459949cdbd377c21ba914c7a, entries=150, sequenceid=84, filesize=11.7 K 2024-12-08T01:15:14,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/4af707910eea47a7af1ed2ae892e3987 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/4af707910eea47a7af1ed2ae892e3987 2024-12-08T01:15:14,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/4af707910eea47a7af1ed2ae892e3987, entries=150, sequenceid=84, filesize=11.7 K 2024-12-08T01:15:14,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 836035c90977e14b643a4b4ed6128d49 in 542ms, sequenceid=84, compaction requested=true 2024-12-08T01:15:14,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:14,011 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:14,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:14,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:14,014 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:14,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:14,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:14,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:14,014 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:14,014 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:14,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:14,014 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:14,014 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/89c7789635e341d991009829bbbe5cf3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/6f0ab344a2a64238932fbc628670aba1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/31a9147bc4d741d3a064c4b0c0d38053, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/eb53ab1aeb6a4ce695702d98786633a8] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=46.9 K 2024-12-08T01:15:14,015 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c7789635e341d991009829bbbe5cf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733620510947 2024-12-08T01:15:14,015 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0ab344a2a64238932fbc628670aba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620511633 2024-12-08T01:15:14,015 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:14,016 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:14,016 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:14,016 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/7da04854b15b4d5faa86bb0f92ed82d4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/243bd9db507f48238f13d0c036c54d8f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4b93dda010e84e36942e700f59f5a62a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d01d0626459949cdbd377c21ba914c7a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=46.9 K 2024-12-08T01:15:14,016 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31a9147bc4d741d3a064c4b0c0d38053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1733620511670 2024-12-08T01:15:14,016 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da04854b15b4d5faa86bb0f92ed82d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733620510947 2024-12-08T01:15:14,017 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb53ab1aeb6a4ce695702d98786633a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733620512322 2024-12-08T01:15:14,017 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 243bd9db507f48238f13d0c036c54d8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620511633 2024-12-08T01:15:14,018 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b93dda010e84e36942e700f59f5a62a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1733620511670 2024-12-08T01:15:14,018 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d01d0626459949cdbd377c21ba914c7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733620512322 2024-12-08T01:15:14,032 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:14,032 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:14,032 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2adc10d8f3b6461f8242d3f3b988884d is 50, key is test_row_0/B:col10/1733620512323/Put/seqid=0 2024-12-08T01:15:14,032 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/1d769449e11b45cfaed212f37211a302 is 50, key is test_row_0/A:col10/1733620512323/Put/seqid=0 2024-12-08T01:15:14,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742070_1246 (size=12139) 2024-12-08T01:15:14,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742071_1247 (size=12139) 2024-12-08T01:15:14,102 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/1d769449e11b45cfaed212f37211a302 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/1d769449e11b45cfaed212f37211a302 2024-12-08T01:15:14,108 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into 1d769449e11b45cfaed212f37211a302(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:14,108 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:14,108 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=12, startTime=1733620514011; duration=0sec 2024-12-08T01:15:14,108 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:14,108 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:14,108 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:14,111 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:14,111 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:14,111 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:14,111 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/92f92f32d2ab455e9110498f89d174de, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5a8da3afccf040cd8e49bf5866439a57, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/46aeb8952c1e470eb443c076a00a842a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/4af707910eea47a7af1ed2ae892e3987] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=46.9 K 2024-12-08T01:15:14,112 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f92f32d2ab455e9110498f89d174de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733620510947 2024-12-08T01:15:14,112 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8da3afccf040cd8e49bf5866439a57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620511633 2024-12-08T01:15:14,112 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46aeb8952c1e470eb443c076a00a842a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1733620511670 2024-12-08T01:15:14,113 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af707910eea47a7af1ed2ae892e3987, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733620512322 2024-12-08T01:15:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:14,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:15:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:14,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3b73f88372934a19a2f04ef4ac1c9995 is 50, key is test_row_0/A:col10/1733620514138/Put/seqid=0 2024-12-08T01:15:14,172 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:14,172 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/178ab11f00e24c9d89c93bd5f9a609a9 is 50, key is test_row_0/C:col10/1733620512323/Put/seqid=0 2024-12-08T01:15:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620574176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620574178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620574182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742072_1248 (size=16681) 2024-12-08T01:15:14,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3b73f88372934a19a2f04ef4ac1c9995 2024-12-08T01:15:14,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/3e346bf0b88e41bc9019300a36e1797c is 50, key is test_row_0/B:col10/1733620514138/Put/seqid=0 2024-12-08T01:15:14,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742073_1249 (size=12139) 2024-12-08T01:15:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620574292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620574292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620574292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742074_1250 (size=12001) 2024-12-08T01:15:14,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620574472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,475 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620574482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,484 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620574502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620574509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620574510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,514 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2adc10d8f3b6461f8242d3f3b988884d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2adc10d8f3b6461f8242d3f3b988884d 2024-12-08T01:15:14,521 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into 2adc10d8f3b6461f8242d3f3b988884d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:14,521 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:14,521 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=12, startTime=1733620514013; duration=0sec 2024-12-08T01:15:14,522 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:14,522 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:14,677 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/178ab11f00e24c9d89c93bd5f9a609a9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/178ab11f00e24c9d89c93bd5f9a609a9 2024-12-08T01:15:14,682 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 178ab11f00e24c9d89c93bd5f9a609a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:14,682 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:14,682 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=12, startTime=1733620514014; duration=0sec 2024-12-08T01:15:14,682 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:14,682 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:14,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/3e346bf0b88e41bc9019300a36e1797c 2024-12-08T01:15:14,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/120aeed6bd6d481da3940b8e5dc27f6c is 50, key is test_row_0/C:col10/1733620514138/Put/seqid=0 2024-12-08T01:15:14,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742075_1251 (size=12001) 2024-12-08T01:15:14,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/120aeed6bd6d481da3940b8e5dc27f6c 2024-12-08T01:15:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3b73f88372934a19a2f04ef4ac1c9995 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3b73f88372934a19a2f04ef4ac1c9995 2024-12-08T01:15:14,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3b73f88372934a19a2f04ef4ac1c9995, entries=250, sequenceid=104, filesize=16.3 K 2024-12-08T01:15:14,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/3e346bf0b88e41bc9019300a36e1797c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3e346bf0b88e41bc9019300a36e1797c 2024-12-08T01:15:14,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3e346bf0b88e41bc9019300a36e1797c, entries=150, sequenceid=104, filesize=11.7 K 2024-12-08T01:15:14,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/120aeed6bd6d481da3940b8e5dc27f6c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/120aeed6bd6d481da3940b8e5dc27f6c 2024-12-08T01:15:14,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/120aeed6bd6d481da3940b8e5dc27f6c, entries=150, sequenceid=104, filesize=11.7 K 2024-12-08T01:15:14,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 836035c90977e14b643a4b4ed6128d49 in 663ms, sequenceid=104, compaction requested=false 2024-12-08T01:15:14,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:14,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:15:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:14,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/94c935518ab349699783be55337f0c75 is 50, key is test_row_0/A:col10/1733620514169/Put/seqid=0 2024-12-08T01:15:14,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620574858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620574865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620574867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742076_1252 (size=12001) 2024-12-08T01:15:14,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/94c935518ab349699783be55337f0c75 2024-12-08T01:15:14,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/8a8b458cac0642958d880178b63ad2c3 is 50, key is test_row_0/B:col10/1733620514169/Put/seqid=0 2024-12-08T01:15:14,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742077_1253 (size=12001) 2024-12-08T01:15:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620574973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620574982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620574982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T01:15:15,122 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:15,122 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-08T01:15:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T01:15:15,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:15,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:15,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:15,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620575182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620575192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620575192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T01:15:15,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T01:15:15,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:15,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/8a8b458cac0642958d880178b63ad2c3 2024-12-08T01:15:15,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/b01b7d18d7b3450197553b2f07d4e9c7 is 50, key is test_row_0/C:col10/1733620514169/Put/seqid=0 2024-12-08T01:15:15,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742078_1254 (size=12001) 2024-12-08T01:15:15,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T01:15:15,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:15,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T01:15:15,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620575491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620575502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620575502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:15,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T01:15:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T01:15:15,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T01:15:15,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/b01b7d18d7b3450197553b2f07d4e9c7 2024-12-08T01:15:15,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/94c935518ab349699783be55337f0c75 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/94c935518ab349699783be55337f0c75 2024-12-08T01:15:15,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/94c935518ab349699783be55337f0c75, entries=150, sequenceid=120, filesize=11.7 K 2024-12-08T01:15:15,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/8a8b458cac0642958d880178b63ad2c3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/8a8b458cac0642958d880178b63ad2c3 2024-12-08T01:15:15,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/8a8b458cac0642958d880178b63ad2c3, entries=150, sequenceid=120, filesize=11.7 K 2024-12-08T01:15:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/b01b7d18d7b3450197553b2f07d4e9c7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/b01b7d18d7b3450197553b2f07d4e9c7 2024-12-08T01:15:15,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/b01b7d18d7b3450197553b2f07d4e9c7, entries=150, sequenceid=120, filesize=11.7 K 2024-12-08T01:15:15,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 836035c90977e14b643a4b4ed6128d49 in 995ms, sequenceid=120, compaction requested=true 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:15,812 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:15,812 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:15,813 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:15,813 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:15,813 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:15,813 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:15,813 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,813 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,813 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2adc10d8f3b6461f8242d3f3b988884d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3e346bf0b88e41bc9019300a36e1797c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/8a8b458cac0642958d880178b63ad2c3] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.3 K 2024-12-08T01:15:15,813 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/1d769449e11b45cfaed212f37211a302, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3b73f88372934a19a2f04ef4ac1c9995, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/94c935518ab349699783be55337f0c75] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=39.9 K 2024-12-08T01:15:15,813 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d769449e11b45cfaed212f37211a302, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733620512322 2024-12-08T01:15:15,813 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2adc10d8f3b6461f8242d3f3b988884d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733620512322 2024-12-08T01:15:15,814 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b73f88372934a19a2f04ef4ac1c9995, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733620513492 2024-12-08T01:15:15,814 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e346bf0b88e41bc9019300a36e1797c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733620513501 2024-12-08T01:15:15,814 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c935518ab349699783be55337f0c75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733620514169 2024-12-08T01:15:15,814 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8b458cac0642958d880178b63ad2c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733620514169 2024-12-08T01:15:15,820 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:15,820 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:15,821 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/3140d5622ee2482ea652ceb5d825a937 is 50, key is test_row_0/B:col10/1733620514169/Put/seqid=0 2024-12-08T01:15:15,821 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/30ad44a04025471f8511c4b0c4c6ab21 is 50, key is test_row_0/A:col10/1733620514169/Put/seqid=0 2024-12-08T01:15:15,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742080_1256 (size=12241) 2024-12-08T01:15:15,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742079_1255 (size=12241) 2024-12-08T01:15:15,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T01:15:15,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:15,890 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T01:15:15,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:15,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:15,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3c0e0285c2574f17883190df21e958a4 is 50, key is test_row_0/A:col10/1733620514858/Put/seqid=0 2024-12-08T01:15:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742081_1257 (size=12151) 2024-12-08T01:15:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:16,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620576035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620576037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620576037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620576142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620576142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620576142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,240 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/3140d5622ee2482ea652ceb5d825a937 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3140d5622ee2482ea652ceb5d825a937 2024-12-08T01:15:16,246 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into 3140d5622ee2482ea652ceb5d825a937(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:16,246 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:16,246 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=13, startTime=1733620515812; duration=0sec 2024-12-08T01:15:16,246 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:16,246 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:16,246 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:16,248 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:16,249 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:16,250 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:16,250 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/178ab11f00e24c9d89c93bd5f9a609a9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/120aeed6bd6d481da3940b8e5dc27f6c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/b01b7d18d7b3450197553b2f07d4e9c7] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.3 K 2024-12-08T01:15:16,251 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/30ad44a04025471f8511c4b0c4c6ab21 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/30ad44a04025471f8511c4b0c4c6ab21 2024-12-08T01:15:16,251 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 178ab11f00e24c9d89c93bd5f9a609a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733620512322 2024-12-08T01:15:16,251 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 120aeed6bd6d481da3940b8e5dc27f6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733620513501 2024-12-08T01:15:16,253 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b01b7d18d7b3450197553b2f07d4e9c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733620514169 2024-12-08T01:15:16,258 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into 30ad44a04025471f8511c4b0c4c6ab21(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:16,258 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:16,258 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=13, startTime=1733620515812; duration=0sec 2024-12-08T01:15:16,258 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:16,258 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T01:15:16,269 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:16,269 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/9b4c05620e3b4c1aae5cecb5e29fb75a is 50, key is test_row_0/C:col10/1733620514169/Put/seqid=0 2024-12-08T01:15:16,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742082_1258 (size=12241) 2024-12-08T01:15:16,289 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/9b4c05620e3b4c1aae5cecb5e29fb75a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/9b4c05620e3b4c1aae5cecb5e29fb75a 2024-12-08T01:15:16,294 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 9b4c05620e3b4c1aae5cecb5e29fb75a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:16,294 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:16,294 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=13, startTime=1733620515812; duration=0sec 2024-12-08T01:15:16,294 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:16,294 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:16,299 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3c0e0285c2574f17883190df21e958a4 2024-12-08T01:15:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2bbd71ba61fc49c696ad6dff3c954dae is 50, key is test_row_0/B:col10/1733620514858/Put/seqid=0 2024-12-08T01:15:16,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742083_1259 (size=12151) 2024-12-08T01:15:16,331 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2bbd71ba61fc49c696ad6dff3c954dae 2024-12-08T01:15:16,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/abee6ec43acd41f9ba9532a306614540 is 50, key is test_row_0/C:col10/1733620514858/Put/seqid=0 2024-12-08T01:15:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620576352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620576352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620576352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742084_1260 (size=12151) 2024-12-08T01:15:16,379 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/abee6ec43acd41f9ba9532a306614540 2024-12-08T01:15:16,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3c0e0285c2574f17883190df21e958a4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3c0e0285c2574f17883190df21e958a4 2024-12-08T01:15:16,393 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3c0e0285c2574f17883190df21e958a4, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T01:15:16,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2bbd71ba61fc49c696ad6dff3c954dae as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2bbd71ba61fc49c696ad6dff3c954dae 2024-12-08T01:15:16,403 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2bbd71ba61fc49c696ad6dff3c954dae, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T01:15:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/abee6ec43acd41f9ba9532a306614540 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/abee6ec43acd41f9ba9532a306614540 2024-12-08T01:15:16,413 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/abee6ec43acd41f9ba9532a306614540, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T01:15:16,414 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 836035c90977e14b643a4b4ed6128d49 in 524ms, sequenceid=139, compaction requested=false 2024-12-08T01:15:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-08T01:15:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-08T01:15:16,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-08T01:15:16,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-12-08T01:15:16,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2960 sec 2024-12-08T01:15:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:16,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:15:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:16,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/8fcd4156ce2d4297af2def991632a339 is 50, key is test_row_0/A:col10/1733620516484/Put/seqid=0 2024-12-08T01:15:16,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742085_1261 (size=12151) 2024-12-08T01:15:16,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/8fcd4156ce2d4297af2def991632a339 2024-12-08T01:15:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/efff42ceb15a4034a66e29e2dff08209 is 50, key is test_row_0/B:col10/1733620516484/Put/seqid=0 2024-12-08T01:15:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620576529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620576530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742086_1262 (size=12151) 2024-12-08T01:15:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620576642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620576642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620576662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620576662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620576662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620576852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620576852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:16,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/efff42ceb15a4034a66e29e2dff08209 2024-12-08T01:15:16,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/d2a1946ff1fb45e1beeeb6673db5a646 is 50, key is test_row_0/C:col10/1733620516484/Put/seqid=0 2024-12-08T01:15:16,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742087_1263 (size=12151) 2024-12-08T01:15:17,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620577161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620577162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620577172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620577182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620577182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T01:15:17,282 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:17,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-08T01:15:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T01:15:17,286 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:17,286 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:17,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:17,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/d2a1946ff1fb45e1beeeb6673db5a646 2024-12-08T01:15:17,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/8fcd4156ce2d4297af2def991632a339 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8fcd4156ce2d4297af2def991632a339 2024-12-08T01:15:17,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8fcd4156ce2d4297af2def991632a339, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T01:15:17,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/efff42ceb15a4034a66e29e2dff08209 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/efff42ceb15a4034a66e29e2dff08209 2024-12-08T01:15:17,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/efff42ceb15a4034a66e29e2dff08209, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T01:15:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T01:15:17,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/d2a1946ff1fb45e1beeeb6673db5a646 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d2a1946ff1fb45e1beeeb6673db5a646 2024-12-08T01:15:17,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d2a1946ff1fb45e1beeeb6673db5a646, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T01:15:17,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 836035c90977e14b643a4b4ed6128d49 in 914ms, sequenceid=156, compaction requested=true 2024-12-08T01:15:17,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:17,401 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:17,403 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:17,403 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:17,403 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:17,403 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:17,403 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/30ad44a04025471f8511c4b0c4c6ab21, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3c0e0285c2574f17883190df21e958a4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8fcd4156ce2d4297af2def991632a339] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.7 K 2024-12-08T01:15:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:17,404 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ad44a04025471f8511c4b0c4c6ab21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733620514169 2024-12-08T01:15:17,405 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:17,405 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:17,405 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:17,405 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3140d5622ee2482ea652ceb5d825a937, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2bbd71ba61fc49c696ad6dff3c954dae, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/efff42ceb15a4034a66e29e2dff08209] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.7 K 2024-12-08T01:15:17,406 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3140d5622ee2482ea652ceb5d825a937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733620514169 2024-12-08T01:15:17,406 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c0e0285c2574f17883190df21e958a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733620514856 2024-12-08T01:15:17,407 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbd71ba61fc49c696ad6dff3c954dae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733620514856 2024-12-08T01:15:17,408 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fcd4156ce2d4297af2def991632a339, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620516035 2024-12-08T01:15:17,408 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting efff42ceb15a4034a66e29e2dff08209, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620516035 2024-12-08T01:15:17,431 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:17,431 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d212615e0bc94d6e9428c334dc0a6a2b is 50, key is test_row_0/B:col10/1733620516484/Put/seqid=0 2024-12-08T01:15:17,439 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:17,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T01:15:17,439 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/ae4182e506bb423697e68429546e4b66 is 50, key is test_row_0/A:col10/1733620516484/Put/seqid=0 2024-12-08T01:15:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:17,439 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:15:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f6ba29252fe64bd297cf172e6f0ac1d9 is 50, key is test_row_0/A:col10/1733620516524/Put/seqid=0 2024-12-08T01:15:17,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742089_1265 (size=12493) 2024-12-08T01:15:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742088_1264 (size=12493) 2024-12-08T01:15:17,483 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/ae4182e506bb423697e68429546e4b66 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/ae4182e506bb423697e68429546e4b66 2024-12-08T01:15:17,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742090_1266 (size=12151) 2024-12-08T01:15:17,509 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into ae4182e506bb423697e68429546e4b66(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:17,509 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f6ba29252fe64bd297cf172e6f0ac1d9 2024-12-08T01:15:17,509 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:17,509 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=13, startTime=1733620517401; duration=0sec 2024-12-08T01:15:17,509 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:17,509 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:17,509 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:17,510 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:17,510 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:17,511 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:17,511 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/9b4c05620e3b4c1aae5cecb5e29fb75a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/abee6ec43acd41f9ba9532a306614540, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d2a1946ff1fb45e1beeeb6673db5a646] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.7 K 2024-12-08T01:15:17,511 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4c05620e3b4c1aae5cecb5e29fb75a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733620514169 2024-12-08T01:15:17,512 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting abee6ec43acd41f9ba9532a306614540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733620514856 2024-12-08T01:15:17,512 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a1946ff1fb45e1beeeb6673db5a646, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620516035 2024-12-08T01:15:17,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/9df3365ffd9243a2898d0272baff0a60 is 50, key is test_row_0/B:col10/1733620516524/Put/seqid=0 2024-12-08T01:15:17,535 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:17,535 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/8036a7f899754c11af84152461c3559d is 50, key is test_row_0/C:col10/1733620516484/Put/seqid=0 2024-12-08T01:15:17,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742091_1267 (size=12151) 2024-12-08T01:15:17,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742092_1268 (size=12493) 2024-12-08T01:15:17,597 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/8036a7f899754c11af84152461c3559d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8036a7f899754c11af84152461c3559d 2024-12-08T01:15:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T01:15:17,603 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 8036a7f899754c11af84152461c3559d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:17,603 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:17,603 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=13, startTime=1733620517403; duration=0sec 2024-12-08T01:15:17,603 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:17,603 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620577710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620577711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620577822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620577822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:17,880 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d212615e0bc94d6e9428c334dc0a6a2b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d212615e0bc94d6e9428c334dc0a6a2b 2024-12-08T01:15:17,886 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into d212615e0bc94d6e9428c334dc0a6a2b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:17,886 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:17,886 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=13, startTime=1733620517403; duration=0sec 2024-12-08T01:15:17,886 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:17,886 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T01:15:17,969 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/9df3365ffd9243a2898d0272baff0a60 2024-12-08T01:15:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/f8f782ca563d490cbdb43c4a19944c61 is 50, key is test_row_0/C:col10/1733620516524/Put/seqid=0 2024-12-08T01:15:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742093_1269 (size=12151) 2024-12-08T01:15:18,010 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/f8f782ca563d490cbdb43c4a19944c61 2024-12-08T01:15:18,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f6ba29252fe64bd297cf172e6f0ac1d9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f6ba29252fe64bd297cf172e6f0ac1d9 2024-12-08T01:15:18,024 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f6ba29252fe64bd297cf172e6f0ac1d9, entries=150, sequenceid=174, filesize=11.9 K 2024-12-08T01:15:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/9df3365ffd9243a2898d0272baff0a60 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9df3365ffd9243a2898d0272baff0a60 2024-12-08T01:15:18,030 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9df3365ffd9243a2898d0272baff0a60, entries=150, sequenceid=174, filesize=11.9 K 2024-12-08T01:15:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/f8f782ca563d490cbdb43c4a19944c61 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f8f782ca563d490cbdb43c4a19944c61 2024-12-08T01:15:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620578032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620578032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,036 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f8f782ca563d490cbdb43c4a19944c61, entries=150, sequenceid=174, filesize=11.9 K 2024-12-08T01:15:18,038 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 836035c90977e14b643a4b4ed6128d49 in 598ms, sequenceid=174, compaction requested=false 2024-12-08T01:15:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-08T01:15:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-08T01:15:18,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-08T01:15:18,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 752 msec 2024-12-08T01:15:18,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 759 msec 2024-12-08T01:15:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:18,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:15:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:18,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/0a1d5d292fc24879bfed740080de9391 is 50, key is test_row_0/A:col10/1733620518182/Put/seqid=0 2024-12-08T01:15:18,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620578209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620578212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620578214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742094_1270 (size=14541) 2024-12-08T01:15:18,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620578322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620578322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620578322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620578341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620578342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T01:15:18,422 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:18,422 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-08T01:15:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T01:15:18,424 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:18,424 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:18,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T01:15:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620578532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620578532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620578532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T01:15:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/0a1d5d292fc24879bfed740080de9391 2024-12-08T01:15:18,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/9a6696f3cfda4d68a964308c39916706 is 50, key is test_row_0/B:col10/1733620518182/Put/seqid=0 2024-12-08T01:15:18,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742095_1271 (size=12151) 2024-12-08T01:15:18,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/9a6696f3cfda4d68a964308c39916706 2024-12-08T01:15:18,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/8969f6c91cb84cbfb6708666755c89d0 is 50, key is test_row_0/C:col10/1733620518182/Put/seqid=0 2024-12-08T01:15:18,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742096_1272 (size=12151) 2024-12-08T01:15:18,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T01:15:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T01:15:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620578842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620578842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620578842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620578852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620578852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:18,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T01:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:18,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T01:15:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T01:15:19,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/8969f6c91cb84cbfb6708666755c89d0 2024-12-08T01:15:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/0a1d5d292fc24879bfed740080de9391 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/0a1d5d292fc24879bfed740080de9391 2024-12-08T01:15:19,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/0a1d5d292fc24879bfed740080de9391, entries=200, sequenceid=192, filesize=14.2 K 2024-12-08T01:15:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/9a6696f3cfda4d68a964308c39916706 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9a6696f3cfda4d68a964308c39916706 2024-12-08T01:15:19,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9a6696f3cfda4d68a964308c39916706, entries=150, sequenceid=192, filesize=11.9 K 2024-12-08T01:15:19,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/8969f6c91cb84cbfb6708666755c89d0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8969f6c91cb84cbfb6708666755c89d0 2024-12-08T01:15:19,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8969f6c91cb84cbfb6708666755c89d0, entries=150, sequenceid=192, filesize=11.9 K 2024-12-08T01:15:19,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 836035c90977e14b643a4b4ed6128d49 in 951ms, sequenceid=192, compaction requested=true 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:19,135 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:19,135 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:19,136 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:19,136 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:19,136 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,137 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d212615e0bc94d6e9428c334dc0a6a2b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9df3365ffd9243a2898d0272baff0a60, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9a6696f3cfda4d68a964308c39916706] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.9 K 2024-12-08T01:15:19,138 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:19,138 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:19,138 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,138 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/ae4182e506bb423697e68429546e4b66, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f6ba29252fe64bd297cf172e6f0ac1d9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/0a1d5d292fc24879bfed740080de9391] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=38.3 K 2024-12-08T01:15:19,138 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d212615e0bc94d6e9428c334dc0a6a2b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620516035 2024-12-08T01:15:19,139 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae4182e506bb423697e68429546e4b66, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620516035 2024-12-08T01:15:19,139 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df3365ffd9243a2898d0272baff0a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620516524 2024-12-08T01:15:19,139 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ba29252fe64bd297cf172e6f0ac1d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620516524 2024-12-08T01:15:19,139 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6696f3cfda4d68a964308c39916706, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733620517702 2024-12-08T01:15:19,139 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a1d5d292fc24879bfed740080de9391, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733620517683 2024-12-08T01:15:19,156 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:19,156 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/25c009eb0f5546f19e8b88ce0952c6c9 is 50, key is test_row_0/A:col10/1733620518182/Put/seqid=0 2024-12-08T01:15:19,163 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:19,164 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d502d81f6db143feb706f8f62caff8a5 is 50, key is test_row_0/B:col10/1733620518182/Put/seqid=0 2024-12-08T01:15:19,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742097_1273 (size=12595) 2024-12-08T01:15:19,184 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/25c009eb0f5546f19e8b88ce0952c6c9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/25c009eb0f5546f19e8b88ce0952c6c9 2024-12-08T01:15:19,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742098_1274 (size=12595) 2024-12-08T01:15:19,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T01:15:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,188 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3155d7a6832640729d598468be862fef is 50, key is test_row_0/A:col10/1733620518210/Put/seqid=0 2024-12-08T01:15:19,195 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into 25c009eb0f5546f19e8b88ce0952c6c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:19,195 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:19,195 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=13, startTime=1733620519135; duration=0sec 2024-12-08T01:15:19,195 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:19,195 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:19,195 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:19,196 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:19,196 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:19,196 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,196 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8036a7f899754c11af84152461c3559d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f8f782ca563d490cbdb43c4a19944c61, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8969f6c91cb84cbfb6708666755c89d0] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=35.9 K 2024-12-08T01:15:19,197 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8036a7f899754c11af84152461c3559d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620516035 2024-12-08T01:15:19,197 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f782ca563d490cbdb43c4a19944c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620516524 2024-12-08T01:15:19,197 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8969f6c91cb84cbfb6708666755c89d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733620517702 2024-12-08T01:15:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742099_1275 (size=9757) 2024-12-08T01:15:19,206 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3155d7a6832640729d598468be862fef 2024-12-08T01:15:19,215 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:19,215 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/77cf6fa8cbc643e6b7549562c1f93a71 is 50, key is test_row_0/C:col10/1733620518182/Put/seqid=0 2024-12-08T01:15:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/5219d5ef244b4d1f8752c50984ee0e33 is 50, key is test_row_0/B:col10/1733620518210/Put/seqid=0 2024-12-08T01:15:19,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742100_1276 (size=12595) 2024-12-08T01:15:19,234 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/77cf6fa8cbc643e6b7549562c1f93a71 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/77cf6fa8cbc643e6b7549562c1f93a71 2024-12-08T01:15:19,241 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 77cf6fa8cbc643e6b7549562c1f93a71(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:19,241 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:19,241 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=13, startTime=1733620519135; duration=0sec 2024-12-08T01:15:19,241 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:19,241 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:19,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742101_1277 (size=9757) 2024-12-08T01:15:19,245 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/5219d5ef244b4d1f8752c50984ee0e33 2024-12-08T01:15:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5dda5eeee1f94ca3867742e8db80b262 is 50, key is test_row_0/C:col10/1733620518210/Put/seqid=0 2024-12-08T01:15:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742102_1278 (size=9757) 2024-12-08T01:15:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:19,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620579367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620579367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620579367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620579482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620579482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620579482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T01:15:19,592 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/d502d81f6db143feb706f8f62caff8a5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d502d81f6db143feb706f8f62caff8a5 2024-12-08T01:15:19,597 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into d502d81f6db143feb706f8f62caff8a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:19,597 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:19,597 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=13, startTime=1733620519135; duration=0sec 2024-12-08T01:15:19,597 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:19,597 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:19,657 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5dda5eeee1f94ca3867742e8db80b262 2024-12-08T01:15:19,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3155d7a6832640729d598468be862fef as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3155d7a6832640729d598468be862fef 2024-12-08T01:15:19,666 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3155d7a6832640729d598468be862fef, entries=100, sequenceid=209, filesize=9.5 K 2024-12-08T01:15:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/5219d5ef244b4d1f8752c50984ee0e33 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5219d5ef244b4d1f8752c50984ee0e33 2024-12-08T01:15:19,672 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5219d5ef244b4d1f8752c50984ee0e33, entries=100, sequenceid=209, filesize=9.5 K 2024-12-08T01:15:19,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5dda5eeee1f94ca3867742e8db80b262 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5dda5eeee1f94ca3867742e8db80b262 2024-12-08T01:15:19,678 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5dda5eeee1f94ca3867742e8db80b262, entries=100, sequenceid=209, filesize=9.5 K 2024-12-08T01:15:19,679 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 836035c90977e14b643a4b4ed6128d49 in 491ms, sequenceid=209, compaction requested=false 2024-12-08T01:15:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-08T01:15:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-08T01:15:19,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-08T01:15:19,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2570 sec 2024-12-08T01:15:19,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2610 sec 2024-12-08T01:15:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:19,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:15:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:19,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/87e1c9204fc54f83948e772d8ffb178a is 50, key is test_row_0/A:col10/1733620519365/Put/seqid=0 2024-12-08T01:15:19,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742103_1279 (size=12151) 2024-12-08T01:15:19,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/87e1c9204fc54f83948e772d8ffb178a 2024-12-08T01:15:19,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620579712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a is 50, key is test_row_0/B:col10/1733620519365/Put/seqid=0 2024-12-08T01:15:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620579715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620579718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742104_1280 (size=12151) 2024-12-08T01:15:19,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a 2024-12-08T01:15:19,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5d7db12d35964f6bad4f20d4e2fbeaa3 is 50, key is test_row_0/C:col10/1733620519365/Put/seqid=0 2024-12-08T01:15:19,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742105_1281 (size=12151) 2024-12-08T01:15:19,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620579822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620579832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620579832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620579872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620579872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:19,874 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:19,874 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620580032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620580042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620580042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5d7db12d35964f6bad4f20d4e2fbeaa3 2024-12-08T01:15:20,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/87e1c9204fc54f83948e772d8ffb178a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/87e1c9204fc54f83948e772d8ffb178a 2024-12-08T01:15:20,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/87e1c9204fc54f83948e772d8ffb178a, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T01:15:20,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a 2024-12-08T01:15:20,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T01:15:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/5d7db12d35964f6bad4f20d4e2fbeaa3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5d7db12d35964f6bad4f20d4e2fbeaa3 2024-12-08T01:15:20,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5d7db12d35964f6bad4f20d4e2fbeaa3, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T01:15:20,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 836035c90977e14b643a4b4ed6128d49 in 482ms, sequenceid=228, compaction requested=true 2024-12-08T01:15:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:20,175 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:20,175 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:20,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:20,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:20,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:20,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:20,178 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:20,178 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:20,178 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,178 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d502d81f6db143feb706f8f62caff8a5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5219d5ef244b4d1f8752c50984ee0e33, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=33.7 K 2024-12-08T01:15:20,178 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:20,178 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:20,178 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,179 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/25c009eb0f5546f19e8b88ce0952c6c9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3155d7a6832640729d598468be862fef, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/87e1c9204fc54f83948e772d8ffb178a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=33.7 K 2024-12-08T01:15:20,179 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25c009eb0f5546f19e8b88ce0952c6c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733620517702 2024-12-08T01:15:20,179 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d502d81f6db143feb706f8f62caff8a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733620517702 2024-12-08T01:15:20,180 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3155d7a6832640729d598468be862fef, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733620518210 2024-12-08T01:15:20,181 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5219d5ef244b4d1f8752c50984ee0e33, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733620518210 2024-12-08T01:15:20,181 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2b96875e8a4e74b6dc7bd3f3dbd25a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620519365 2024-12-08T01:15:20,182 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e1c9204fc54f83948e772d8ffb178a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620519365 2024-12-08T01:15:20,198 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:20,199 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/7edea1f2ae5d40d6910e9594a5352e7f is 50, key is test_row_0/A:col10/1733620519365/Put/seqid=0 2024-12-08T01:15:20,199 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:20,200 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/063476c1bb874c72a81b9041c4a0130d is 50, key is test_row_0/B:col10/1733620519365/Put/seqid=0 2024-12-08T01:15:20,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742107_1283 (size=12697) 2024-12-08T01:15:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742106_1282 (size=12697) 2024-12-08T01:15:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:20,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:15:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:20,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/de32e36d5e234a76a898ac18dd078baa is 50, key is test_row_0/A:col10/1733620519714/Put/seqid=0 2024-12-08T01:15:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742108_1284 (size=12151) 2024-12-08T01:15:20,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620580372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620580372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620580376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620580482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620580482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620580492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T01:15:20,572 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:20,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-08T01:15:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:20,574 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:20,575 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:20,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:20,621 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/7edea1f2ae5d40d6910e9594a5352e7f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/7edea1f2ae5d40d6910e9594a5352e7f 2024-12-08T01:15:20,630 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into 7edea1f2ae5d40d6910e9594a5352e7f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:20,630 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:20,630 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=13, startTime=1733620520175; duration=0sec 2024-12-08T01:15:20,630 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:20,630 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:20,630 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:20,633 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:20,633 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:20,633 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,633 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/77cf6fa8cbc643e6b7549562c1f93a71, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5dda5eeee1f94ca3867742e8db80b262, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5d7db12d35964f6bad4f20d4e2fbeaa3] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=33.7 K 2024-12-08T01:15:20,634 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77cf6fa8cbc643e6b7549562c1f93a71, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733620517702 2024-12-08T01:15:20,634 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/063476c1bb874c72a81b9041c4a0130d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/063476c1bb874c72a81b9041c4a0130d 2024-12-08T01:15:20,635 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dda5eeee1f94ca3867742e8db80b262, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733620518210 2024-12-08T01:15:20,635 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d7db12d35964f6bad4f20d4e2fbeaa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620519365 2024-12-08T01:15:20,641 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into 063476c1bb874c72a81b9041c4a0130d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:20,641 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:20,641 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=13, startTime=1733620520175; duration=0sec 2024-12-08T01:15:20,641 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:20,641 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:20,644 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:20,644 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/39658eae8bea45bc97d57fb39947bea2 is 50, key is test_row_0/C:col10/1733620519365/Put/seqid=0 2024-12-08T01:15:20,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742109_1285 (size=12697) 2024-12-08T01:15:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620580692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620580692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620580702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:20,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:20,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/de32e36d5e234a76a898ac18dd078baa 2024-12-08T01:15:20,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2ff04bba0b4841578e462df5b07bdc3b is 50, key is test_row_0/B:col10/1733620519714/Put/seqid=0 2024-12-08T01:15:20,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742110_1286 (size=12151) 2024-12-08T01:15:20,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:20,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:21,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620581001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620581001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620581012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:21,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,071 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/39658eae8bea45bc97d57fb39947bea2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/39658eae8bea45bc97d57fb39947bea2 2024-12-08T01:15:21,076 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 39658eae8bea45bc97d57fb39947bea2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:21,076 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:21,076 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=13, startTime=1733620520176; duration=0sec 2024-12-08T01:15:21,076 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:21,076 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:21,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2ff04bba0b4841578e462df5b07bdc3b 2024-12-08T01:15:21,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/c1312be5ccb04d50829343cd97c55227 is 50, key is test_row_0/C:col10/1733620519714/Put/seqid=0 2024-12-08T01:15:21,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:21,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742111_1287 (size=12151) 2024-12-08T01:15:21,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:21,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:21,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620581512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620581512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620581522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/c1312be5ccb04d50829343cd97c55227 2024-12-08T01:15:21,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/de32e36d5e234a76a898ac18dd078baa as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/de32e36d5e234a76a898ac18dd078baa 2024-12-08T01:15:21,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/de32e36d5e234a76a898ac18dd078baa, entries=150, sequenceid=246, filesize=11.9 K 2024-12-08T01:15:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/2ff04bba0b4841578e462df5b07bdc3b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2ff04bba0b4841578e462df5b07bdc3b 2024-12-08T01:15:21,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2ff04bba0b4841578e462df5b07bdc3b, entries=150, sequenceid=246, filesize=11.9 K 2024-12-08T01:15:21,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/c1312be5ccb04d50829343cd97c55227 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c1312be5ccb04d50829343cd97c55227 2024-12-08T01:15:21,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c1312be5ccb04d50829343cd97c55227, entries=150, sequenceid=246, filesize=11.9 K 2024-12-08T01:15:21,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 836035c90977e14b643a4b4ed6128d49 in 1272ms, sequenceid=246, compaction requested=false 2024-12-08T01:15:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:21,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T01:15:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:21,648 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:15:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:21,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f5c0130eb38d450d922f542498c9ab26 is 50, key is test_row_0/A:col10/1733620520366/Put/seqid=0 2024-12-08T01:15:21,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742112_1288 (size=12251) 2024-12-08T01:15:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:21,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:21,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620581919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:21,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620581921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620582032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620582032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,056 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f5c0130eb38d450d922f542498c9ab26 2024-12-08T01:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/484cf667424f4f6f8027c8a25616313e is 50, key is test_row_0/B:col10/1733620520366/Put/seqid=0 2024-12-08T01:15:22,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742113_1289 (size=12251) 2024-12-08T01:15:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620582242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620582242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,466 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/484cf667424f4f6f8027c8a25616313e 2024-12-08T01:15:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/3a44161d33584bb7922d77c793b8e603 is 50, key is test_row_0/C:col10/1733620520366/Put/seqid=0 2024-12-08T01:15:22,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742114_1290 (size=12251) 2024-12-08T01:15:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620582522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,524 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620582532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620582532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,536 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,536 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620582552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620582552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:22,878 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/3a44161d33584bb7922d77c793b8e603 2024-12-08T01:15:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f5c0130eb38d450d922f542498c9ab26 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f5c0130eb38d450d922f542498c9ab26 2024-12-08T01:15:22,885 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f5c0130eb38d450d922f542498c9ab26, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:15:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/484cf667424f4f6f8027c8a25616313e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/484cf667424f4f6f8027c8a25616313e 2024-12-08T01:15:22,889 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/484cf667424f4f6f8027c8a25616313e, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:15:22,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/3a44161d33584bb7922d77c793b8e603 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/3a44161d33584bb7922d77c793b8e603 2024-12-08T01:15:22,892 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/3a44161d33584bb7922d77c793b8e603, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:15:22,893 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 836035c90977e14b643a4b4ed6128d49 in 1245ms, sequenceid=263, compaction requested=true 2024-12-08T01:15:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-08T01:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-08T01:15:22,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-08T01:15:22,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3220 sec 2024-12-08T01:15:22,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.3270 sec 2024-12-08T01:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:23,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T01:15:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:23,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3db08a9752d445438a3c8f3361e85234 is 50, key is test_row_0/A:col10/1733620523062/Put/seqid=0 2024-12-08T01:15:23,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742115_1291 (size=14741) 2024-12-08T01:15:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620583081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620583085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620583192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620583192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620583401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620583401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3db08a9752d445438a3c8f3361e85234 2024-12-08T01:15:23,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/41a583ca37104bb9aab1d490db03c909 is 50, key is test_row_0/B:col10/1733620523062/Put/seqid=0 2024-12-08T01:15:23,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742116_1292 (size=12301) 2024-12-08T01:15:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620583712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620583712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:23,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/41a583ca37104bb9aab1d490db03c909 2024-12-08T01:15:23,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/a35e9596946f4f1c8252c84882c51181 is 50, key is test_row_0/C:col10/1733620523062/Put/seqid=0 2024-12-08T01:15:23,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742117_1293 (size=12301) 2024-12-08T01:15:23,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/a35e9596946f4f1c8252c84882c51181 2024-12-08T01:15:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/3db08a9752d445438a3c8f3361e85234 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3db08a9752d445438a3c8f3361e85234 2024-12-08T01:15:23,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3db08a9752d445438a3c8f3361e85234, entries=200, sequenceid=283, filesize=14.4 K 2024-12-08T01:15:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/41a583ca37104bb9aab1d490db03c909 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/41a583ca37104bb9aab1d490db03c909 2024-12-08T01:15:23,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/41a583ca37104bb9aab1d490db03c909, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T01:15:23,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/a35e9596946f4f1c8252c84882c51181 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a35e9596946f4f1c8252c84882c51181 2024-12-08T01:15:23,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a35e9596946f4f1c8252c84882c51181, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T01:15:23,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 836035c90977e14b643a4b4ed6128d49 in 862ms, sequenceid=283, compaction requested=true 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:23,926 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:23,926 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:23,927 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51840 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:23,927 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:23,927 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:23,927 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:23,927 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:23,927 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:23,927 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/7edea1f2ae5d40d6910e9594a5352e7f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/de32e36d5e234a76a898ac18dd078baa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f5c0130eb38d450d922f542498c9ab26, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3db08a9752d445438a3c8f3361e85234] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=50.6 K 2024-12-08T01:15:23,927 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/063476c1bb874c72a81b9041c4a0130d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2ff04bba0b4841578e462df5b07bdc3b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/484cf667424f4f6f8027c8a25616313e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/41a583ca37104bb9aab1d490db03c909] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.2 K 2024-12-08T01:15:23,928 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7edea1f2ae5d40d6910e9594a5352e7f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620519365 2024-12-08T01:15:23,928 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 063476c1bb874c72a81b9041c4a0130d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620519365 2024-12-08T01:15:23,928 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting de32e36d5e234a76a898ac18dd078baa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733620519709 2024-12-08T01:15:23,928 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff04bba0b4841578e462df5b07bdc3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733620519709 2024-12-08T01:15:23,928 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 484cf667424f4f6f8027c8a25616313e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620520354 2024-12-08T01:15:23,929 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c0130eb38d450d922f542498c9ab26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620520354 2024-12-08T01:15:23,929 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a583ca37104bb9aab1d490db03c909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733620521916 2024-12-08T01:15:23,930 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3db08a9752d445438a3c8f3361e85234, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733620521916 2024-12-08T01:15:23,949 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:23,950 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/66a0d3501b5347d1a69c1408be06be23 is 50, key is test_row_0/B:col10/1733620523062/Put/seqid=0 2024-12-08T01:15:23,959 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:23,960 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/c6c14eb159334aaf945ffa1eefc6ccd3 is 50, key is test_row_0/A:col10/1733620523062/Put/seqid=0 2024-12-08T01:15:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742118_1294 (size=12983) 2024-12-08T01:15:24,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742119_1295 (size=12983) 2024-12-08T01:15:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:24,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T01:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:24,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/a32f559d51084dc784f42aff36421ac0 is 50, key is test_row_0/A:col10/1733620523084/Put/seqid=0 2024-12-08T01:15:24,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742120_1296 (size=14741) 2024-12-08T01:15:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620584271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620584271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620584382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620584382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/66a0d3501b5347d1a69c1408be06be23 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/66a0d3501b5347d1a69c1408be06be23 2024-12-08T01:15:24,400 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into 66a0d3501b5347d1a69c1408be06be23(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:24,400 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:24,400 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=12, startTime=1733620523926; duration=0sec 2024-12-08T01:15:24,401 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:24,401 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:24,401 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:24,402 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:24,402 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:24,403 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:24,403 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/39658eae8bea45bc97d57fb39947bea2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c1312be5ccb04d50829343cd97c55227, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/3a44161d33584bb7922d77c793b8e603, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a35e9596946f4f1c8252c84882c51181] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.2 K 2024-12-08T01:15:24,403 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 39658eae8bea45bc97d57fb39947bea2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620519365 2024-12-08T01:15:24,403 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c1312be5ccb04d50829343cd97c55227, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733620519709 2024-12-08T01:15:24,404 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a44161d33584bb7922d77c793b8e603, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620520354 2024-12-08T01:15:24,404 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a35e9596946f4f1c8252c84882c51181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733620521916 2024-12-08T01:15:24,415 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/c6c14eb159334aaf945ffa1eefc6ccd3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/c6c14eb159334aaf945ffa1eefc6ccd3 2024-12-08T01:15:24,416 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:24,416 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/670fb3ac5c3b4a7ca981210a5bf88f06 is 50, key is test_row_0/C:col10/1733620523062/Put/seqid=0 2024-12-08T01:15:24,420 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into c6c14eb159334aaf945ffa1eefc6ccd3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:24,420 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:24,420 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=12, startTime=1733620523926; duration=0sec 2024-12-08T01:15:24,420 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:24,420 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:24,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742121_1297 (size=12983) 2024-12-08T01:15:24,449 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/670fb3ac5c3b4a7ca981210a5bf88f06 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/670fb3ac5c3b4a7ca981210a5bf88f06 2024-12-08T01:15:24,453 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 670fb3ac5c3b4a7ca981210a5bf88f06(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:24,453 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:24,453 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=12, startTime=1733620523926; duration=0sec 2024-12-08T01:15:24,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:24,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:24,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620584532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,535 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620584542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,544 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620584552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,555 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620584593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620584593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/a32f559d51084dc784f42aff36421ac0 2024-12-08T01:15:24,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/69cd248b38bc47f393429e181628122b is 50, key is test_row_0/B:col10/1733620523084/Put/seqid=0 2024-12-08T01:15:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742122_1298 (size=12301) 2024-12-08T01:15:24,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/69cd248b38bc47f393429e181628122b 2024-12-08T01:15:24,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/7e5a3975dc5d48918c87d79409fe3826 is 50, key is test_row_0/C:col10/1733620523084/Put/seqid=0 2024-12-08T01:15:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742123_1299 (size=12301) 2024-12-08T01:15:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T01:15:24,742 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:24,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-08T01:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T01:15:24,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:24,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:24,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T01:15:24,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T01:15:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:24,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:24,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620584902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620584902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:24,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:15:25,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T01:15:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:25,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T01:15:25,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/7e5a3975dc5d48918c87d79409fe3826 2024-12-08T01:15:25,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/a32f559d51084dc784f42aff36421ac0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a32f559d51084dc784f42aff36421ac0 2024-12-08T01:15:25,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a32f559d51084dc784f42aff36421ac0, entries=200, sequenceid=296, filesize=14.4 K 2024-12-08T01:15:25,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/69cd248b38bc47f393429e181628122b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/69cd248b38bc47f393429e181628122b 2024-12-08T01:15:25,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/69cd248b38bc47f393429e181628122b, entries=150, sequenceid=296, filesize=12.0 K 2024-12-08T01:15:25,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/7e5a3975dc5d48918c87d79409fe3826 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/7e5a3975dc5d48918c87d79409fe3826 2024-12-08T01:15:25,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/7e5a3975dc5d48918c87d79409fe3826, entries=150, sequenceid=296, filesize=12.0 K 2024-12-08T01:15:25,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 836035c90977e14b643a4b4ed6128d49 in 892ms, sequenceid=296, compaction requested=false 2024-12-08T01:15:25,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:25,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T01:15:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:25,203 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:15:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:25,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f33f23f74a0140b6bc39fb8336498faa is 50, key is test_row_0/A:col10/1733620524269/Put/seqid=0 2024-12-08T01:15:25,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742124_1300 (size=12301) 2024-12-08T01:15:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T01:15:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:25,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620585428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620585431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:25,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620585542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620585542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:25,611 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f33f23f74a0140b6bc39fb8336498faa 2024-12-08T01:15:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/b26372d75ddc48d2b04dd7ec948cf068 is 50, key is test_row_0/B:col10/1733620524269/Put/seqid=0 2024-12-08T01:15:25,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742125_1301 (size=12301) 2024-12-08T01:15:25,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620585751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:25,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620585751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T01:15:26,021 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/b26372d75ddc48d2b04dd7ec948cf068 2024-12-08T01:15:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/a497e9cf8bc94b84945f67cc2cd5989d is 50, key is test_row_0/C:col10/1733620524269/Put/seqid=0 2024-12-08T01:15:26,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742126_1302 (size=12301) 2024-12-08T01:15:26,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620586061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620586062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,431 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/a497e9cf8bc94b84945f67cc2cd5989d 2024-12-08T01:15:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/f33f23f74a0140b6bc39fb8336498faa as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f33f23f74a0140b6bc39fb8336498faa 2024-12-08T01:15:26,440 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f33f23f74a0140b6bc39fb8336498faa, entries=150, sequenceid=319, filesize=12.0 K 2024-12-08T01:15:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/b26372d75ddc48d2b04dd7ec948cf068 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/b26372d75ddc48d2b04dd7ec948cf068 2024-12-08T01:15:26,446 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/b26372d75ddc48d2b04dd7ec948cf068, entries=150, sequenceid=319, filesize=12.0 K 2024-12-08T01:15:26,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/a497e9cf8bc94b84945f67cc2cd5989d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a497e9cf8bc94b84945f67cc2cd5989d 2024-12-08T01:15:26,451 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a497e9cf8bc94b84945f67cc2cd5989d, entries=150, sequenceid=319, filesize=12.0 K 2024-12-08T01:15:26,452 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 836035c90977e14b643a4b4ed6128d49 in 1248ms, sequenceid=319, compaction requested=true 2024-12-08T01:15:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-08T01:15:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-08T01:15:26,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-08T01:15:26,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7080 sec 2024-12-08T01:15:26,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7160 sec 2024-12-08T01:15:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:26,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:15:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:26,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/aec3731cf196430393fa63589de2f86f is 50, key is test_row_0/A:col10/1733620525430/Put/seqid=0 2024-12-08T01:15:26,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742127_1303 (size=14741) 2024-12-08T01:15:26,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/aec3731cf196430393fa63589de2f86f 2024-12-08T01:15:26,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/86c50d55a95041afb2432a9029bbbca6 is 50, key is test_row_0/B:col10/1733620525430/Put/seqid=0 2024-12-08T01:15:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620586609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742128_1304 (size=12301) 2024-12-08T01:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620586614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620586722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620586722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T01:15:26,902 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:26,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-08T01:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T01:15:26,905 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:26,905 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:26,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620586932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620586932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T01:15:27,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/86c50d55a95041afb2432a9029bbbca6 2024-12-08T01:15:27,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/29a172432d95470ca9f7a8a022bf694d is 50, key is test_row_0/C:col10/1733620525430/Put/seqid=0 2024-12-08T01:15:27,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742129_1305 (size=12301) 2024-12-08T01:15:27,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T01:15:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T01:15:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T01:15:27,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620587242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:27,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620587242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T01:15:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:27,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/29a172432d95470ca9f7a8a022bf694d 2024-12-08T01:15:27,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/aec3731cf196430393fa63589de2f86f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/aec3731cf196430393fa63589de2f86f 2024-12-08T01:15:27,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/aec3731cf196430393fa63589de2f86f, entries=200, sequenceid=333, filesize=14.4 K 2024-12-08T01:15:27,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/86c50d55a95041afb2432a9029bbbca6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/86c50d55a95041afb2432a9029bbbca6 2024-12-08T01:15:27,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/86c50d55a95041afb2432a9029bbbca6, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T01:15:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/29a172432d95470ca9f7a8a022bf694d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/29a172432d95470ca9f7a8a022bf694d 2024-12-08T01:15:27,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/29a172432d95470ca9f7a8a022bf694d, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T01:15:27,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 836035c90977e14b643a4b4ed6128d49 in 879ms, sequenceid=333, compaction requested=true 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:27,454 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:27,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:27,455 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:27,455 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:27,455 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:27,455 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:27,455 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,455 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,455 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/c6c14eb159334aaf945ffa1eefc6ccd3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a32f559d51084dc784f42aff36421ac0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f33f23f74a0140b6bc39fb8336498faa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/aec3731cf196430393fa63589de2f86f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=53.5 K 2024-12-08T01:15:27,455 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/66a0d3501b5347d1a69c1408be06be23, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/69cd248b38bc47f393429e181628122b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/b26372d75ddc48d2b04dd7ec948cf068, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/86c50d55a95041afb2432a9029bbbca6] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.7 K 2024-12-08T01:15:27,456 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c14eb159334aaf945ffa1eefc6ccd3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733620521916 2024-12-08T01:15:27,456 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a0d3501b5347d1a69c1408be06be23, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733620521916 2024-12-08T01:15:27,456 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32f559d51084dc784f42aff36421ac0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733620523075 2024-12-08T01:15:27,456 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 69cd248b38bc47f393429e181628122b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733620523079 2024-12-08T01:15:27,457 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33f23f74a0140b6bc39fb8336498faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733620524261 2024-12-08T01:15:27,457 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b26372d75ddc48d2b04dd7ec948cf068, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733620524261 2024-12-08T01:15:27,457 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec3731cf196430393fa63589de2f86f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733620525427 2024-12-08T01:15:27,457 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c50d55a95041afb2432a9029bbbca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733620525427 2024-12-08T01:15:27,472 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:27,472 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/a5eba25912f54f00a5cd251e1cafbb6a is 50, key is test_row_0/A:col10/1733620525430/Put/seqid=0 2024-12-08T01:15:27,476 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:27,476 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/5024f98171a6488f8e2fb1cfe3f1005c is 50, key is test_row_0/B:col10/1733620525430/Put/seqid=0 2024-12-08T01:15:27,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742131_1307 (size=13119) 2024-12-08T01:15:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742130_1306 (size=13119) 2024-12-08T01:15:27,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T01:15:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,516 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:15:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:27,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/df2e0068347241b19333ba3b2d27f4ad is 50, key is test_row_0/A:col10/1733620526607/Put/seqid=0 2024-12-08T01:15:27,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742132_1308 (size=12301) 2024-12-08T01:15:27,529 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/df2e0068347241b19333ba3b2d27f4ad 2024-12-08T01:15:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T01:15:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/22e5751e95d9448190ad1b1edcdc9f2f is 50, key is test_row_0/B:col10/1733620526607/Put/seqid=0 2024-12-08T01:15:27,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742133_1309 (size=12301) 2024-12-08T01:15:27,542 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/22e5751e95d9448190ad1b1edcdc9f2f 2024-12-08T01:15:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/758a365b2ccd4e81bb33126326c7aa5b is 50, key is test_row_0/C:col10/1733620526607/Put/seqid=0 2024-12-08T01:15:27,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742134_1310 (size=12301) 2024-12-08T01:15:27,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620587772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620587776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620587882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620587882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:27,899 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/5024f98171a6488f8e2fb1cfe3f1005c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5024f98171a6488f8e2fb1cfe3f1005c 2024-12-08T01:15:27,903 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/a5eba25912f54f00a5cd251e1cafbb6a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a5eba25912f54f00a5cd251e1cafbb6a 2024-12-08T01:15:27,907 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into 5024f98171a6488f8e2fb1cfe3f1005c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:27,907 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:27,907 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=12, startTime=1733620527454; duration=0sec 2024-12-08T01:15:27,907 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:27,907 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:27,907 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:27,909 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:27,909 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:27,909 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,909 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/670fb3ac5c3b4a7ca981210a5bf88f06, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/7e5a3975dc5d48918c87d79409fe3826, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a497e9cf8bc94b84945f67cc2cd5989d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/29a172432d95470ca9f7a8a022bf694d] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.7 K 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 670fb3ac5c3b4a7ca981210a5bf88f06, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733620521916 2024-12-08T01:15:27,910 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into a5eba25912f54f00a5cd251e1cafbb6a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:27,910 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=12, startTime=1733620527454; duration=0sec 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5a3975dc5d48918c87d79409fe3826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733620523079 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a497e9cf8bc94b84945f67cc2cd5989d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733620524261 2024-12-08T01:15:27,910 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a172432d95470ca9f7a8a022bf694d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733620525427 2024-12-08T01:15:27,918 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:27,919 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/43ea32e2e4d541d298fce27b782dfe70 is 50, key is test_row_0/C:col10/1733620525430/Put/seqid=0 2024-12-08T01:15:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742135_1311 (size=13119) 2024-12-08T01:15:27,946 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/43ea32e2e4d541d298fce27b782dfe70 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/43ea32e2e4d541d298fce27b782dfe70 2024-12-08T01:15:27,953 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 43ea32e2e4d541d298fce27b782dfe70(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:27,953 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:27,953 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=12, startTime=1733620527454; duration=0sec 2024-12-08T01:15:27,953 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:27,953 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:27,958 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/758a365b2ccd4e81bb33126326c7aa5b 2024-12-08T01:15:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/df2e0068347241b19333ba3b2d27f4ad as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/df2e0068347241b19333ba3b2d27f4ad 2024-12-08T01:15:27,977 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/df2e0068347241b19333ba3b2d27f4ad, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T01:15:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/22e5751e95d9448190ad1b1edcdc9f2f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/22e5751e95d9448190ad1b1edcdc9f2f 2024-12-08T01:15:27,983 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/22e5751e95d9448190ad1b1edcdc9f2f, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T01:15:27,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/758a365b2ccd4e81bb33126326c7aa5b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/758a365b2ccd4e81bb33126326c7aa5b 2024-12-08T01:15:27,994 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/758a365b2ccd4e81bb33126326c7aa5b, entries=150, sequenceid=351, filesize=12.0 K 2024-12-08T01:15:27,996 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 836035c90977e14b643a4b4ed6128d49 in 480ms, sequenceid=351, compaction requested=false 2024-12-08T01:15:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-08T01:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-08T01:15:28,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-08T01:15:28,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0920 sec 2024-12-08T01:15:28,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0980 sec 2024-12-08T01:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T01:15:28,042 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:28,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-08T01:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T01:15:28,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:28,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:28,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:28,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:28,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:28,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:28,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:28,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/4605530007724878b437de0cc85318c8 is 50, key is test_row_0/A:col10/1733620527771/Put/seqid=0 2024-12-08T01:15:28,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742136_1312 (size=12301) 2024-12-08T01:15:28,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620588132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620588133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T01:15:28,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620588242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620588242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T01:15:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620588452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620588452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:28,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:28,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/4605530007724878b437de0cc85318c8 2024-12-08T01:15:28,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/016104d1a52843ada1d28458f54844f9 is 50, key is test_row_0/B:col10/1733620527771/Put/seqid=0 2024-12-08T01:15:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742137_1313 (size=12301) 2024-12-08T01:15:28,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733620588552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,554 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50628 deadline: 1733620588561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,563 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50642 deadline: 1733620588582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,585 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 836035c90977e14b643a4b4ed6128d49 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T01:15:28,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620588762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620588762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:28,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/016104d1a52843ada1d28458f54844f9 2024-12-08T01:15:28,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/d0fe50a7cef144b19c0f9b8ede6b94ac is 50, key is test_row_0/C:col10/1733620527771/Put/seqid=0 2024-12-08T01:15:28,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742138_1314 (size=12301) 2024-12-08T01:15:28,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/d0fe50a7cef144b19c0f9b8ede6b94ac 2024-12-08T01:15:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/4605530007724878b437de0cc85318c8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/4605530007724878b437de0cc85318c8 2024-12-08T01:15:28,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/4605530007724878b437de0cc85318c8, entries=150, sequenceid=370, filesize=12.0 K 2024-12-08T01:15:28,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:28,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:28,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/016104d1a52843ada1d28458f54844f9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/016104d1a52843ada1d28458f54844f9 2024-12-08T01:15:28,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/016104d1a52843ada1d28458f54844f9, entries=150, sequenceid=370, filesize=12.0 K 2024-12-08T01:15:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/d0fe50a7cef144b19c0f9b8ede6b94ac as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d0fe50a7cef144b19c0f9b8ede6b94ac 2024-12-08T01:15:28,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d0fe50a7cef144b19c0f9b8ede6b94ac, entries=150, sequenceid=370, filesize=12.0 K 2024-12-08T01:15:28,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 836035c90977e14b643a4b4ed6128d49 in 883ms, sequenceid=370, compaction requested=true 2024-12-08T01:15:28,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:28,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:28,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:28,978 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:28,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:28,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:28,979 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:28,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:28,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:28,981 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:28,981 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:28,981 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:28,981 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:28,981 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,981 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:28,981 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a5eba25912f54f00a5cd251e1cafbb6a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/df2e0068347241b19333ba3b2d27f4ad, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/4605530007724878b437de0cc85318c8] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=36.8 K 2024-12-08T01:15:28,981 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5024f98171a6488f8e2fb1cfe3f1005c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/22e5751e95d9448190ad1b1edcdc9f2f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/016104d1a52843ada1d28458f54844f9] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=36.8 K 2024-12-08T01:15:28,982 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5eba25912f54f00a5cd251e1cafbb6a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733620525427 2024-12-08T01:15:28,982 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5024f98171a6488f8e2fb1cfe3f1005c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733620525427 2024-12-08T01:15:28,982 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting df2e0068347241b19333ba3b2d27f4ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733620526588 2024-12-08T01:15:28,982 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e5751e95d9448190ad1b1edcdc9f2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733620526588 2024-12-08T01:15:28,982 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4605530007724878b437de0cc85318c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733620527771 2024-12-08T01:15:28,983 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 016104d1a52843ada1d28458f54844f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733620527771 2024-12-08T01:15:28,994 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:28,995 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/8c978098c806404fac3c23a8e4906056 is 50, key is test_row_0/A:col10/1733620527771/Put/seqid=0 2024-12-08T01:15:28,997 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#273 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:28,997 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/afcfff4b9b2340bc9fe427e26e3a9090 is 50, key is test_row_0/B:col10/1733620527771/Put/seqid=0 2024-12-08T01:15:29,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742139_1315 (size=13221) 2024-12-08T01:15:29,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742140_1316 (size=13221) 2024-12-08T01:15:29,016 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/afcfff4b9b2340bc9fe427e26e3a9090 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/afcfff4b9b2340bc9fe427e26e3a9090 2024-12-08T01:15:29,022 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into afcfff4b9b2340bc9fe427e26e3a9090(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:29,022 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:29,022 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=13, startTime=1733620528978; duration=0sec 2024-12-08T01:15:29,022 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:29,022 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:29,022 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:29,023 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:29,024 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:29,024 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:29,024 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/43ea32e2e4d541d298fce27b782dfe70, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/758a365b2ccd4e81bb33126326c7aa5b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d0fe50a7cef144b19c0f9b8ede6b94ac] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=36.8 K 2024-12-08T01:15:29,025 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ea32e2e4d541d298fce27b782dfe70, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733620525427 2024-12-08T01:15:29,025 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 758a365b2ccd4e81bb33126326c7aa5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733620526588 2024-12-08T01:15:29,025 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d0fe50a7cef144b19c0f9b8ede6b94ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733620527771 2024-12-08T01:15:29,042 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:29,043 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/20de4adaea98430b9cd009b92f21fe4f is 50, key is test_row_0/C:col10/1733620527771/Put/seqid=0 2024-12-08T01:15:29,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742141_1317 (size=13221) 2024-12-08T01:15:29,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:29,116 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/fbb7e1233015473d9e0d0d36440d7dd9 is 50, key is test_row_0/A:col10/1733620528131/Put/seqid=0 2024-12-08T01:15:29,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742142_1318 (size=12301) 2024-12-08T01:15:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T01:15:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:29,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:29,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620589293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620589296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620589401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620589402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,414 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/8c978098c806404fac3c23a8e4906056 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8c978098c806404fac3c23a8e4906056 2024-12-08T01:15:29,419 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into 8c978098c806404fac3c23a8e4906056(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:29,419 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:29,419 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=13, startTime=1733620528978; duration=0sec 2024-12-08T01:15:29,419 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:29,419 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:29,468 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/20de4adaea98430b9cd009b92f21fe4f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/20de4adaea98430b9cd009b92f21fe4f 2024-12-08T01:15:29,472 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into 20de4adaea98430b9cd009b92f21fe4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:29,472 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:29,472 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=13, startTime=1733620528979; duration=0sec 2024-12-08T01:15:29,472 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:29,472 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:29,537 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/fbb7e1233015473d9e0d0d36440d7dd9 2024-12-08T01:15:29,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/fec79fb3a2c1494fadcfbb12eca86d84 is 50, key is test_row_0/B:col10/1733620528131/Put/seqid=0 2024-12-08T01:15:29,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742143_1319 (size=12301) 2024-12-08T01:15:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620589611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620589611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620589922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620589922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:29,948 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/fec79fb3a2c1494fadcfbb12eca86d84 2024-12-08T01:15:29,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/e7746b432a784da59c5938aae8a626e5 is 50, key is test_row_0/C:col10/1733620528131/Put/seqid=0 2024-12-08T01:15:29,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742144_1320 (size=12301) 2024-12-08T01:15:29,959 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/e7746b432a784da59c5938aae8a626e5 2024-12-08T01:15:29,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/fbb7e1233015473d9e0d0d36440d7dd9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb7e1233015473d9e0d0d36440d7dd9 2024-12-08T01:15:29,965 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb7e1233015473d9e0d0d36440d7dd9, entries=150, sequenceid=387, filesize=12.0 K 2024-12-08T01:15:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/fec79fb3a2c1494fadcfbb12eca86d84 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/fec79fb3a2c1494fadcfbb12eca86d84 2024-12-08T01:15:29,969 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/fec79fb3a2c1494fadcfbb12eca86d84, entries=150, sequenceid=387, filesize=12.0 K 2024-12-08T01:15:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/e7746b432a784da59c5938aae8a626e5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/e7746b432a784da59c5938aae8a626e5 2024-12-08T01:15:29,973 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/e7746b432a784da59c5938aae8a626e5, entries=150, sequenceid=387, filesize=12.0 K 2024-12-08T01:15:29,974 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 836035c90977e14b643a4b4ed6128d49 in 858ms, sequenceid=387, compaction requested=false 2024-12-08T01:15:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-08T01:15:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-08T01:15:29,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-08T01:15:29,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9300 sec 2024-12-08T01:15:29,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.9360 sec 2024-12-08T01:15:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T01:15:30,192 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:30,193 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-08T01:15:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T01:15:30,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:30,195 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:30,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T01:15:30,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T01:15:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:30,347 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:15:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:30,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/dd5b47f91c3a448ebb23c8e5f8f14165 is 50, key is test_row_0/A:col10/1733620529292/Put/seqid=0 2024-12-08T01:15:30,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742145_1321 (size=12301) 2024-12-08T01:15:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:30,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. as already flushing 2024-12-08T01:15:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620590465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620590466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T01:15:30,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620590571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620590571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:30,756 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/dd5b47f91c3a448ebb23c8e5f8f14165 2024-12-08T01:15:30,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/1b5c7e5ef76e49a8aaf4870c4d7892ec is 50, key is test_row_0/B:col10/1733620529292/Put/seqid=0 2024-12-08T01:15:30,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742146_1322 (size=12301) 2024-12-08T01:15:30,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620590782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620590782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T01:15:30,944 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-12-08T01:15:30,944 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:30,944 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,944 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,944 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:30,945 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-12-08T01:15:30,945 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:30,945 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,945 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:30,946 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-12-08T01:15:30,946 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:30,946 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,946 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:30,947 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-12-08T01:15:30,947 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:30,947 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,947 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:30,947 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-12-08T01:15:30,948 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:30,948 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,948 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:30,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:31,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:31,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620591093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620591092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:31,166 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/1b5c7e5ef76e49a8aaf4870c4d7892ec 2024-12-08T01:15:31,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/636d100687c8426882f1dd9d03a97679 is 50, key is test_row_0/C:col10/1733620529292/Put/seqid=0 2024-12-08T01:15:31,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742147_1323 (size=12301) 2024-12-08T01:15:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T01:15:31,583 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/636d100687c8426882f1dd9d03a97679 2024-12-08T01:15:31,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/dd5b47f91c3a448ebb23c8e5f8f14165 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/dd5b47f91c3a448ebb23c8e5f8f14165 2024-12-08T01:15:31,599 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/dd5b47f91c3a448ebb23c8e5f8f14165, entries=150, sequenceid=405, filesize=12.0 K 2024-12-08T01:15:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/1b5c7e5ef76e49a8aaf4870c4d7892ec as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/1b5c7e5ef76e49a8aaf4870c4d7892ec 2024-12-08T01:15:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50620 deadline: 1733620591602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50588 deadline: 1733620591602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 2024-12-08T01:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:15:31,603 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/1b5c7e5ef76e49a8aaf4870c4d7892ec, entries=150, sequenceid=405, filesize=12.0 K 2024-12-08T01:15:31,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/636d100687c8426882f1dd9d03a97679 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/636d100687c8426882f1dd9d03a97679 2024-12-08T01:15:31,607 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/636d100687c8426882f1dd9d03a97679, entries=150, sequenceid=405, filesize=12.0 K 2024-12-08T01:15:31,607 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 836035c90977e14b643a4b4ed6128d49 in 1260ms, sequenceid=405, compaction requested=true 2024-12-08T01:15:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-08T01:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-08T01:15:31,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-08T01:15:31,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-12-08T01:15:31,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.4170 sec 2024-12-08T01:15:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T01:15:32,343 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:32,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T01:15:32,616 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-12-08T01:15:32,616 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-12-08T01:15:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:32,616 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:32,616 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:32,616 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:32,616 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:32,616 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:32,616 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:32,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:32,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:32,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:32,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:32,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/5974ce94d2aa474fa3f967d8208854f1 is 50, key is test_row_0/A:col10/1733620532613/Put/seqid=0 2024-12-08T01:15:32,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742148_1324 (size=12301) 2024-12-08T01:15:33,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/5974ce94d2aa474fa3f967d8208854f1 2024-12-08T01:15:33,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/18a0fec5164149c8bc50db9b0f3d9b7a is 50, key is test_row_0/B:col10/1733620532613/Put/seqid=0 2024-12-08T01:15:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742149_1325 (size=12301) 2024-12-08T01:15:33,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/18a0fec5164149c8bc50db9b0f3d9b7a 2024-12-08T01:15:33,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/55bf2e0e7d9446c6a3efd155d191f31c is 50, key is test_row_0/C:col10/1733620532613/Put/seqid=0 2024-12-08T01:15:33,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742150_1326 (size=12301) 2024-12-08T01:15:33,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/55bf2e0e7d9446c6a3efd155d191f31c 2024-12-08T01:15:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/5974ce94d2aa474fa3f967d8208854f1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/5974ce94d2aa474fa3f967d8208854f1 2024-12-08T01:15:33,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/5974ce94d2aa474fa3f967d8208854f1, entries=150, sequenceid=424, filesize=12.0 K 2024-12-08T01:15:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/18a0fec5164149c8bc50db9b0f3d9b7a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/18a0fec5164149c8bc50db9b0f3d9b7a 2024-12-08T01:15:33,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/18a0fec5164149c8bc50db9b0f3d9b7a, entries=150, sequenceid=424, filesize=12.0 K 2024-12-08T01:15:33,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/55bf2e0e7d9446c6a3efd155d191f31c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/55bf2e0e7d9446c6a3efd155d191f31c 2024-12-08T01:15:33,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/55bf2e0e7d9446c6a3efd155d191f31c, entries=150, sequenceid=424, filesize=12.0 K 2024-12-08T01:15:33,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 836035c90977e14b643a4b4ed6128d49 in 871ms, sequenceid=424, compaction requested=true 2024-12-08T01:15:33,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:33,487 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:33,487 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 836035c90977e14b643a4b4ed6128d49:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/A is initiating minor compaction (all files) 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/B is initiating minor compaction (all files) 2024-12-08T01:15:33,488 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/B in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:33,488 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/A in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:33,488 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/afcfff4b9b2340bc9fe427e26e3a9090, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/fec79fb3a2c1494fadcfbb12eca86d84, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/1b5c7e5ef76e49a8aaf4870c4d7892ec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/18a0fec5164149c8bc50db9b0f3d9b7a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.9 K 2024-12-08T01:15:33,488 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8c978098c806404fac3c23a8e4906056, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb7e1233015473d9e0d0d36440d7dd9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/dd5b47f91c3a448ebb23c8e5f8f14165, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/5974ce94d2aa474fa3f967d8208854f1] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.9 K 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting afcfff4b9b2340bc9fe427e26e3a9090, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733620527771 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c978098c806404fac3c23a8e4906056, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733620527771 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb7e1233015473d9e0d0d36440d7dd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733620528125 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fec79fb3a2c1494fadcfbb12eca86d84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733620528125 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5b47f91c3a448ebb23c8e5f8f14165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733620529286 2024-12-08T01:15:33,488 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b5c7e5ef76e49a8aaf4870c4d7892ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733620529286 2024-12-08T01:15:33,489 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5974ce94d2aa474fa3f967d8208854f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733620530457 2024-12-08T01:15:33,489 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a0fec5164149c8bc50db9b0f3d9b7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733620530457 2024-12-08T01:15:33,495 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:33,495 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:33,495 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/d687640319c440998e2ff0d5b40d25f9 is 50, key is test_row_0/A:col10/1733620532613/Put/seqid=0 2024-12-08T01:15:33,495 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/f5d525b25d814ee48f5c6a427cced60f is 50, key is test_row_0/B:col10/1733620532613/Put/seqid=0 2024-12-08T01:15:33,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742151_1327 (size=13357) 2024-12-08T01:15:33,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742152_1328 (size=13357) 2024-12-08T01:15:33,908 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/d687640319c440998e2ff0d5b40d25f9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/d687640319c440998e2ff0d5b40d25f9 2024-12-08T01:15:33,908 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/f5d525b25d814ee48f5c6a427cced60f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/f5d525b25d814ee48f5c6a427cced60f 2024-12-08T01:15:33,912 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/A of 836035c90977e14b643a4b4ed6128d49 into d687640319c440998e2ff0d5b40d25f9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:33,912 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/B of 836035c90977e14b643a4b4ed6128d49 into f5d525b25d814ee48f5c6a427cced60f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:33,913 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/A, priority=12, startTime=1733620533486; duration=0sec 2024-12-08T01:15:33,913 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/B, priority=12, startTime=1733620533487; duration=0sec 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:A 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:B 2024-12-08T01:15:33,913 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:33,914 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:33,914 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 836035c90977e14b643a4b4ed6128d49/C is initiating minor compaction (all files) 2024-12-08T01:15:33,914 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 836035c90977e14b643a4b4ed6128d49/C in TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:33,914 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/20de4adaea98430b9cd009b92f21fe4f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/e7746b432a784da59c5938aae8a626e5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/636d100687c8426882f1dd9d03a97679, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/55bf2e0e7d9446c6a3efd155d191f31c] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp, totalSize=48.9 K 2024-12-08T01:15:33,915 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20de4adaea98430b9cd009b92f21fe4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733620527771 2024-12-08T01:15:33,915 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7746b432a784da59c5938aae8a626e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733620528125 2024-12-08T01:15:33,915 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636d100687c8426882f1dd9d03a97679, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733620529286 2024-12-08T01:15:33,915 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55bf2e0e7d9446c6a3efd155d191f31c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733620530457 2024-12-08T01:15:33,923 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 836035c90977e14b643a4b4ed6128d49#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:33,923 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/f1199f6c72f94d11960df5b0d2f95ea9 is 50, key is test_row_0/C:col10/1733620532613/Put/seqid=0 2024-12-08T01:15:33,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742153_1329 (size=13357) 2024-12-08T01:15:34,335 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/f1199f6c72f94d11960df5b0d2f95ea9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f1199f6c72f94d11960df5b0d2f95ea9 2024-12-08T01:15:34,341 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 836035c90977e14b643a4b4ed6128d49/C of 836035c90977e14b643a4b4ed6128d49 into f1199f6c72f94d11960df5b0d2f95ea9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:34,341 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 836035c90977e14b643a4b4ed6128d49: 2024-12-08T01:15:34,341 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., storeName=836035c90977e14b643a4b4ed6128d49/C, priority=12, startTime=1733620533487; duration=0sec 2024-12-08T01:15:34,341 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:34,341 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 836035c90977e14b643a4b4ed6128d49:C 2024-12-08T01:15:38,614 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-12-08T01:15:38,614 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:38,614 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,614 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:38,625 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-12-08T01:15:38,625 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:38,625 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,625 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:38,663 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-12-08T01:15:38,663 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:15:38,663 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,663 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3816 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3874 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3791 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3818 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3861 2024-12-08T01:15:38,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T01:15:38,664 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:15:38,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:15:38,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:38,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:15:38,665 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T01:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T01:15:38,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620538667"}]},"ts":"1733620538667"} 2024-12-08T01:15:38,669 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T01:15:38,669 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T01:15:38,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:15:38,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, UNASSIGN}] 2024-12-08T01:15:38,672 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, UNASSIGN 2024-12-08T01:15:38,672 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=836035c90977e14b643a4b4ed6128d49, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:38,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, UNASSIGN because future has completed 2024-12-08T01:15:38,674 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:15:38,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630}] 2024-12-08T01:15:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T01:15:38,826 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:38,826 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 836035c90977e14b643a4b4ed6128d49, disabling compactions & flushes 2024-12-08T01:15:38,827 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. after waiting 0 ms 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:38,827 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 836035c90977e14b643a4b4ed6128d49 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=A 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=B 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 836035c90977e14b643a4b4ed6128d49, store=C 2024-12-08T01:15:38,827 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:38,831 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/fbb30f400b64417a9f4641042efe1957 is 50, key is test_row_1/A:col10/1733620538662/Put/seqid=0 2024-12-08T01:15:38,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742154_1330 (size=9857) 2024-12-08T01:15:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T01:15:39,235 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/fbb30f400b64417a9f4641042efe1957 2024-12-08T01:15:39,246 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/e0eacb15bbc140f58dee3fd3796559e2 is 50, key is test_row_1/B:col10/1733620538662/Put/seqid=0 2024-12-08T01:15:39,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742155_1331 (size=9857) 2024-12-08T01:15:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T01:15:39,653 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/e0eacb15bbc140f58dee3fd3796559e2 2024-12-08T01:15:39,659 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/c262fec3a394492db10c6b729586fba5 is 50, key is test_row_1/C:col10/1733620538662/Put/seqid=0 2024-12-08T01:15:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742156_1332 (size=9857) 2024-12-08T01:15:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T01:15:40,062 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/c262fec3a394492db10c6b729586fba5 2024-12-08T01:15:40,066 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/A/fbb30f400b64417a9f4641042efe1957 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb30f400b64417a9f4641042efe1957 2024-12-08T01:15:40,070 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb30f400b64417a9f4641042efe1957, entries=100, sequenceid=433, filesize=9.6 K 2024-12-08T01:15:40,071 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/B/e0eacb15bbc140f58dee3fd3796559e2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/e0eacb15bbc140f58dee3fd3796559e2 2024-12-08T01:15:40,075 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/e0eacb15bbc140f58dee3fd3796559e2, entries=100, sequenceid=433, filesize=9.6 K 2024-12-08T01:15:40,076 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/.tmp/C/c262fec3a394492db10c6b729586fba5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c262fec3a394492db10c6b729586fba5 2024-12-08T01:15:40,080 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c262fec3a394492db10c6b729586fba5, entries=100, sequenceid=433, filesize=9.6 K 2024-12-08T01:15:40,081 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 836035c90977e14b643a4b4ed6128d49 in 1254ms, sequenceid=433, compaction requested=false 2024-12-08T01:15:40,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/89c7789635e341d991009829bbbe5cf3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/6f0ab344a2a64238932fbc628670aba1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/31a9147bc4d741d3a064c4b0c0d38053, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/1d769449e11b45cfaed212f37211a302, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/eb53ab1aeb6a4ce695702d98786633a8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3b73f88372934a19a2f04ef4ac1c9995, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/30ad44a04025471f8511c4b0c4c6ab21, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/94c935518ab349699783be55337f0c75, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3c0e0285c2574f17883190df21e958a4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/ae4182e506bb423697e68429546e4b66, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8fcd4156ce2d4297af2def991632a339, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f6ba29252fe64bd297cf172e6f0ac1d9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/0a1d5d292fc24879bfed740080de9391, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/25c009eb0f5546f19e8b88ce0952c6c9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3155d7a6832640729d598468be862fef, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/7edea1f2ae5d40d6910e9594a5352e7f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/87e1c9204fc54f83948e772d8ffb178a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/de32e36d5e234a76a898ac18dd078baa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f5c0130eb38d450d922f542498c9ab26, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3db08a9752d445438a3c8f3361e85234, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/c6c14eb159334aaf945ffa1eefc6ccd3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a32f559d51084dc784f42aff36421ac0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f33f23f74a0140b6bc39fb8336498faa, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/aec3731cf196430393fa63589de2f86f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a5eba25912f54f00a5cd251e1cafbb6a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/df2e0068347241b19333ba3b2d27f4ad, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8c978098c806404fac3c23a8e4906056, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/4605530007724878b437de0cc85318c8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb7e1233015473d9e0d0d36440d7dd9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/dd5b47f91c3a448ebb23c8e5f8f14165, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/5974ce94d2aa474fa3f967d8208854f1] to archive 2024-12-08T01:15:40,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:40,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/89c7789635e341d991009829bbbe5cf3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/89c7789635e341d991009829bbbe5cf3 2024-12-08T01:15:40,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/6f0ab344a2a64238932fbc628670aba1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/6f0ab344a2a64238932fbc628670aba1 2024-12-08T01:15:40,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/31a9147bc4d741d3a064c4b0c0d38053 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/31a9147bc4d741d3a064c4b0c0d38053 2024-12-08T01:15:40,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/1d769449e11b45cfaed212f37211a302 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/1d769449e11b45cfaed212f37211a302 2024-12-08T01:15:40,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/eb53ab1aeb6a4ce695702d98786633a8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/eb53ab1aeb6a4ce695702d98786633a8 2024-12-08T01:15:40,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3b73f88372934a19a2f04ef4ac1c9995 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3b73f88372934a19a2f04ef4ac1c9995 2024-12-08T01:15:40,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/30ad44a04025471f8511c4b0c4c6ab21 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/30ad44a04025471f8511c4b0c4c6ab21 2024-12-08T01:15:40,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/94c935518ab349699783be55337f0c75 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/94c935518ab349699783be55337f0c75 2024-12-08T01:15:40,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3c0e0285c2574f17883190df21e958a4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3c0e0285c2574f17883190df21e958a4 2024-12-08T01:15:40,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/ae4182e506bb423697e68429546e4b66 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/ae4182e506bb423697e68429546e4b66 2024-12-08T01:15:40,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8fcd4156ce2d4297af2def991632a339 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8fcd4156ce2d4297af2def991632a339 2024-12-08T01:15:40,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f6ba29252fe64bd297cf172e6f0ac1d9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f6ba29252fe64bd297cf172e6f0ac1d9 2024-12-08T01:15:40,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/0a1d5d292fc24879bfed740080de9391 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/0a1d5d292fc24879bfed740080de9391 2024-12-08T01:15:40,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/25c009eb0f5546f19e8b88ce0952c6c9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/25c009eb0f5546f19e8b88ce0952c6c9 2024-12-08T01:15:40,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3155d7a6832640729d598468be862fef to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3155d7a6832640729d598468be862fef 2024-12-08T01:15:40,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/7edea1f2ae5d40d6910e9594a5352e7f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/7edea1f2ae5d40d6910e9594a5352e7f 2024-12-08T01:15:40,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/87e1c9204fc54f83948e772d8ffb178a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/87e1c9204fc54f83948e772d8ffb178a 2024-12-08T01:15:40,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/de32e36d5e234a76a898ac18dd078baa to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/de32e36d5e234a76a898ac18dd078baa 2024-12-08T01:15:40,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f5c0130eb38d450d922f542498c9ab26 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f5c0130eb38d450d922f542498c9ab26 2024-12-08T01:15:40,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3db08a9752d445438a3c8f3361e85234 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/3db08a9752d445438a3c8f3361e85234 2024-12-08T01:15:40,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/c6c14eb159334aaf945ffa1eefc6ccd3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/c6c14eb159334aaf945ffa1eefc6ccd3 2024-12-08T01:15:40,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a32f559d51084dc784f42aff36421ac0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a32f559d51084dc784f42aff36421ac0 2024-12-08T01:15:40,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f33f23f74a0140b6bc39fb8336498faa to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/f33f23f74a0140b6bc39fb8336498faa 2024-12-08T01:15:40,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/aec3731cf196430393fa63589de2f86f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/aec3731cf196430393fa63589de2f86f 2024-12-08T01:15:40,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a5eba25912f54f00a5cd251e1cafbb6a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/a5eba25912f54f00a5cd251e1cafbb6a 2024-12-08T01:15:40,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/df2e0068347241b19333ba3b2d27f4ad to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/df2e0068347241b19333ba3b2d27f4ad 2024-12-08T01:15:40,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8c978098c806404fac3c23a8e4906056 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/8c978098c806404fac3c23a8e4906056 2024-12-08T01:15:40,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/4605530007724878b437de0cc85318c8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/4605530007724878b437de0cc85318c8 2024-12-08T01:15:40,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb7e1233015473d9e0d0d36440d7dd9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb7e1233015473d9e0d0d36440d7dd9 2024-12-08T01:15:40,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/dd5b47f91c3a448ebb23c8e5f8f14165 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/dd5b47f91c3a448ebb23c8e5f8f14165 2024-12-08T01:15:40,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/5974ce94d2aa474fa3f967d8208854f1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/5974ce94d2aa474fa3f967d8208854f1 2024-12-08T01:15:40,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/7da04854b15b4d5faa86bb0f92ed82d4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/243bd9db507f48238f13d0c036c54d8f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4b93dda010e84e36942e700f59f5a62a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2adc10d8f3b6461f8242d3f3b988884d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d01d0626459949cdbd377c21ba914c7a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3e346bf0b88e41bc9019300a36e1797c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3140d5622ee2482ea652ceb5d825a937, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/8a8b458cac0642958d880178b63ad2c3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2bbd71ba61fc49c696ad6dff3c954dae, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d212615e0bc94d6e9428c334dc0a6a2b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/efff42ceb15a4034a66e29e2dff08209, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9df3365ffd9243a2898d0272baff0a60, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d502d81f6db143feb706f8f62caff8a5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9a6696f3cfda4d68a964308c39916706, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5219d5ef244b4d1f8752c50984ee0e33, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/063476c1bb874c72a81b9041c4a0130d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2ff04bba0b4841578e462df5b07bdc3b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/484cf667424f4f6f8027c8a25616313e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/66a0d3501b5347d1a69c1408be06be23, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/41a583ca37104bb9aab1d490db03c909, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/69cd248b38bc47f393429e181628122b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/b26372d75ddc48d2b04dd7ec948cf068, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5024f98171a6488f8e2fb1cfe3f1005c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/86c50d55a95041afb2432a9029bbbca6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/22e5751e95d9448190ad1b1edcdc9f2f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/afcfff4b9b2340bc9fe427e26e3a9090, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/016104d1a52843ada1d28458f54844f9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/fec79fb3a2c1494fadcfbb12eca86d84, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/1b5c7e5ef76e49a8aaf4870c4d7892ec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/18a0fec5164149c8bc50db9b0f3d9b7a] to archive 2024-12-08T01:15:40,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:40,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/7da04854b15b4d5faa86bb0f92ed82d4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/7da04854b15b4d5faa86bb0f92ed82d4 2024-12-08T01:15:40,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/243bd9db507f48238f13d0c036c54d8f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/243bd9db507f48238f13d0c036c54d8f 2024-12-08T01:15:40,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4b93dda010e84e36942e700f59f5a62a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4b93dda010e84e36942e700f59f5a62a 2024-12-08T01:15:40,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2adc10d8f3b6461f8242d3f3b988884d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2adc10d8f3b6461f8242d3f3b988884d 2024-12-08T01:15:40,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d01d0626459949cdbd377c21ba914c7a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d01d0626459949cdbd377c21ba914c7a 2024-12-08T01:15:40,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3e346bf0b88e41bc9019300a36e1797c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3e346bf0b88e41bc9019300a36e1797c 2024-12-08T01:15:40,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3140d5622ee2482ea652ceb5d825a937 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/3140d5622ee2482ea652ceb5d825a937 2024-12-08T01:15:40,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/8a8b458cac0642958d880178b63ad2c3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/8a8b458cac0642958d880178b63ad2c3 2024-12-08T01:15:40,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2bbd71ba61fc49c696ad6dff3c954dae to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2bbd71ba61fc49c696ad6dff3c954dae 2024-12-08T01:15:40,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d212615e0bc94d6e9428c334dc0a6a2b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d212615e0bc94d6e9428c334dc0a6a2b 2024-12-08T01:15:40,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/efff42ceb15a4034a66e29e2dff08209 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/efff42ceb15a4034a66e29e2dff08209 2024-12-08T01:15:40,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9df3365ffd9243a2898d0272baff0a60 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9df3365ffd9243a2898d0272baff0a60 2024-12-08T01:15:40,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d502d81f6db143feb706f8f62caff8a5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/d502d81f6db143feb706f8f62caff8a5 2024-12-08T01:15:40,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9a6696f3cfda4d68a964308c39916706 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/9a6696f3cfda4d68a964308c39916706 2024-12-08T01:15:40,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5219d5ef244b4d1f8752c50984ee0e33 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5219d5ef244b4d1f8752c50984ee0e33 2024-12-08T01:15:40,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/063476c1bb874c72a81b9041c4a0130d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/063476c1bb874c72a81b9041c4a0130d 2024-12-08T01:15:40,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/4c2b96875e8a4e74b6dc7bd3f3dbd25a 2024-12-08T01:15:40,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2ff04bba0b4841578e462df5b07bdc3b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/2ff04bba0b4841578e462df5b07bdc3b 2024-12-08T01:15:40,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/484cf667424f4f6f8027c8a25616313e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/484cf667424f4f6f8027c8a25616313e 2024-12-08T01:15:40,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/66a0d3501b5347d1a69c1408be06be23 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/66a0d3501b5347d1a69c1408be06be23 2024-12-08T01:15:40,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/41a583ca37104bb9aab1d490db03c909 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/41a583ca37104bb9aab1d490db03c909 2024-12-08T01:15:40,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/69cd248b38bc47f393429e181628122b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/69cd248b38bc47f393429e181628122b 2024-12-08T01:15:40,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/b26372d75ddc48d2b04dd7ec948cf068 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/b26372d75ddc48d2b04dd7ec948cf068 2024-12-08T01:15:40,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5024f98171a6488f8e2fb1cfe3f1005c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/5024f98171a6488f8e2fb1cfe3f1005c 2024-12-08T01:15:40,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/86c50d55a95041afb2432a9029bbbca6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/86c50d55a95041afb2432a9029bbbca6 2024-12-08T01:15:40,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/22e5751e95d9448190ad1b1edcdc9f2f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/22e5751e95d9448190ad1b1edcdc9f2f 2024-12-08T01:15:40,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/afcfff4b9b2340bc9fe427e26e3a9090 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/afcfff4b9b2340bc9fe427e26e3a9090 2024-12-08T01:15:40,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/016104d1a52843ada1d28458f54844f9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/016104d1a52843ada1d28458f54844f9 2024-12-08T01:15:40,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/fec79fb3a2c1494fadcfbb12eca86d84 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/fec79fb3a2c1494fadcfbb12eca86d84 2024-12-08T01:15:40,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/1b5c7e5ef76e49a8aaf4870c4d7892ec to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/1b5c7e5ef76e49a8aaf4870c4d7892ec 2024-12-08T01:15:40,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/18a0fec5164149c8bc50db9b0f3d9b7a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/18a0fec5164149c8bc50db9b0f3d9b7a 2024-12-08T01:15:40,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/92f92f32d2ab455e9110498f89d174de, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5a8da3afccf040cd8e49bf5866439a57, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/46aeb8952c1e470eb443c076a00a842a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/178ab11f00e24c9d89c93bd5f9a609a9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/4af707910eea47a7af1ed2ae892e3987, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/120aeed6bd6d481da3940b8e5dc27f6c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/9b4c05620e3b4c1aae5cecb5e29fb75a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/b01b7d18d7b3450197553b2f07d4e9c7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/abee6ec43acd41f9ba9532a306614540, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8036a7f899754c11af84152461c3559d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d2a1946ff1fb45e1beeeb6673db5a646, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f8f782ca563d490cbdb43c4a19944c61, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/77cf6fa8cbc643e6b7549562c1f93a71, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8969f6c91cb84cbfb6708666755c89d0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5dda5eeee1f94ca3867742e8db80b262, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/39658eae8bea45bc97d57fb39947bea2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5d7db12d35964f6bad4f20d4e2fbeaa3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c1312be5ccb04d50829343cd97c55227, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/3a44161d33584bb7922d77c793b8e603, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/670fb3ac5c3b4a7ca981210a5bf88f06, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a35e9596946f4f1c8252c84882c51181, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/7e5a3975dc5d48918c87d79409fe3826, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a497e9cf8bc94b84945f67cc2cd5989d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/43ea32e2e4d541d298fce27b782dfe70, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/29a172432d95470ca9f7a8a022bf694d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/758a365b2ccd4e81bb33126326c7aa5b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/20de4adaea98430b9cd009b92f21fe4f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d0fe50a7cef144b19c0f9b8ede6b94ac, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/e7746b432a784da59c5938aae8a626e5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/636d100687c8426882f1dd9d03a97679, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/55bf2e0e7d9446c6a3efd155d191f31c] to archive 2024-12-08T01:15:40,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:40,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/92f92f32d2ab455e9110498f89d174de to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/92f92f32d2ab455e9110498f89d174de 2024-12-08T01:15:40,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5a8da3afccf040cd8e49bf5866439a57 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5a8da3afccf040cd8e49bf5866439a57 2024-12-08T01:15:40,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/46aeb8952c1e470eb443c076a00a842a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/46aeb8952c1e470eb443c076a00a842a 2024-12-08T01:15:40,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/178ab11f00e24c9d89c93bd5f9a609a9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/178ab11f00e24c9d89c93bd5f9a609a9 2024-12-08T01:15:40,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/4af707910eea47a7af1ed2ae892e3987 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/4af707910eea47a7af1ed2ae892e3987 2024-12-08T01:15:40,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/120aeed6bd6d481da3940b8e5dc27f6c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/120aeed6bd6d481da3940b8e5dc27f6c 2024-12-08T01:15:40,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/9b4c05620e3b4c1aae5cecb5e29fb75a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/9b4c05620e3b4c1aae5cecb5e29fb75a 2024-12-08T01:15:40,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/b01b7d18d7b3450197553b2f07d4e9c7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/b01b7d18d7b3450197553b2f07d4e9c7 2024-12-08T01:15:40,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/abee6ec43acd41f9ba9532a306614540 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/abee6ec43acd41f9ba9532a306614540 2024-12-08T01:15:40,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8036a7f899754c11af84152461c3559d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8036a7f899754c11af84152461c3559d 2024-12-08T01:15:40,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d2a1946ff1fb45e1beeeb6673db5a646 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d2a1946ff1fb45e1beeeb6673db5a646 2024-12-08T01:15:40,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f8f782ca563d490cbdb43c4a19944c61 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f8f782ca563d490cbdb43c4a19944c61 2024-12-08T01:15:40,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/77cf6fa8cbc643e6b7549562c1f93a71 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/77cf6fa8cbc643e6b7549562c1f93a71 2024-12-08T01:15:40,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8969f6c91cb84cbfb6708666755c89d0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/8969f6c91cb84cbfb6708666755c89d0 2024-12-08T01:15:40,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5dda5eeee1f94ca3867742e8db80b262 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5dda5eeee1f94ca3867742e8db80b262 2024-12-08T01:15:40,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/39658eae8bea45bc97d57fb39947bea2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/39658eae8bea45bc97d57fb39947bea2 2024-12-08T01:15:40,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5d7db12d35964f6bad4f20d4e2fbeaa3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/5d7db12d35964f6bad4f20d4e2fbeaa3 2024-12-08T01:15:40,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c1312be5ccb04d50829343cd97c55227 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c1312be5ccb04d50829343cd97c55227 2024-12-08T01:15:40,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/3a44161d33584bb7922d77c793b8e603 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/3a44161d33584bb7922d77c793b8e603 2024-12-08T01:15:40,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/670fb3ac5c3b4a7ca981210a5bf88f06 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/670fb3ac5c3b4a7ca981210a5bf88f06 2024-12-08T01:15:40,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a35e9596946f4f1c8252c84882c51181 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a35e9596946f4f1c8252c84882c51181 2024-12-08T01:15:40,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/7e5a3975dc5d48918c87d79409fe3826 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/7e5a3975dc5d48918c87d79409fe3826 2024-12-08T01:15:40,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a497e9cf8bc94b84945f67cc2cd5989d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/a497e9cf8bc94b84945f67cc2cd5989d 2024-12-08T01:15:40,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/43ea32e2e4d541d298fce27b782dfe70 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/43ea32e2e4d541d298fce27b782dfe70 2024-12-08T01:15:40,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/29a172432d95470ca9f7a8a022bf694d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/29a172432d95470ca9f7a8a022bf694d 2024-12-08T01:15:40,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/758a365b2ccd4e81bb33126326c7aa5b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/758a365b2ccd4e81bb33126326c7aa5b 2024-12-08T01:15:40,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/20de4adaea98430b9cd009b92f21fe4f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/20de4adaea98430b9cd009b92f21fe4f 2024-12-08T01:15:40,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d0fe50a7cef144b19c0f9b8ede6b94ac to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/d0fe50a7cef144b19c0f9b8ede6b94ac 2024-12-08T01:15:40,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/e7746b432a784da59c5938aae8a626e5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/e7746b432a784da59c5938aae8a626e5 2024-12-08T01:15:40,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/636d100687c8426882f1dd9d03a97679 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/636d100687c8426882f1dd9d03a97679 2024-12-08T01:15:40,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/55bf2e0e7d9446c6a3efd155d191f31c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/55bf2e0e7d9446c6a3efd155d191f31c 2024-12-08T01:15:40,190 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=1 2024-12-08T01:15:40,191 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. 2024-12-08T01:15:40,191 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 836035c90977e14b643a4b4ed6128d49: Waiting for close lock at 1733620538827Running coprocessor pre-close hooks at 1733620538827Disabling compacts and flushes for region at 1733620538827Disabling writes for close at 1733620538827Obtaining lock to block concurrent updates at 1733620538827Preparing flush snapshotting stores in 836035c90977e14b643a4b4ed6128d49 at 1733620538827Finished memstore snapshotting TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733620538827Flushing stores of TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49. at 1733620538828 (+1 ms)Flushing 836035c90977e14b643a4b4ed6128d49/A: creating writer at 1733620538828Flushing 836035c90977e14b643a4b4ed6128d49/A: appending metadata at 1733620538830 (+2 ms)Flushing 836035c90977e14b643a4b4ed6128d49/A: closing flushed file at 1733620538831 (+1 ms)Flushing 836035c90977e14b643a4b4ed6128d49/B: creating writer at 1733620539242 (+411 ms)Flushing 836035c90977e14b643a4b4ed6128d49/B: appending metadata at 1733620539246 (+4 ms)Flushing 836035c90977e14b643a4b4ed6128d49/B: closing flushed file at 1733620539246Flushing 836035c90977e14b643a4b4ed6128d49/C: creating writer at 1733620539656 (+410 ms)Flushing 836035c90977e14b643a4b4ed6128d49/C: appending metadata at 1733620539658 (+2 ms)Flushing 836035c90977e14b643a4b4ed6128d49/C: closing flushed file at 1733620539658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7049581d: reopening flushed file at 1733620540065 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ea4f6bc: reopening flushed file at 1733620540070 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e20c34: reopening flushed file at 1733620540075 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 836035c90977e14b643a4b4ed6128d49 in 1254ms, sequenceid=433, compaction requested=false at 1733620540081 (+6 ms)Writing region close event to WAL at 1733620540187 (+106 ms)Running coprocessor post-close hooks at 1733620540191 (+4 ms)Closed at 1733620540191 2024-12-08T01:15:40,193 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:40,193 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=836035c90977e14b643a4b4ed6128d49, regionState=CLOSED 2024-12-08T01:15:40,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:15:40,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-08T01:15:40,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 836035c90977e14b643a4b4ed6128d49, server=96982154c400,44843,1733620436630 in 1.5210 sec 2024-12-08T01:15:40,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-08T01:15:40,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=836035c90977e14b643a4b4ed6128d49, UNASSIGN in 1.5260 sec 2024-12-08T01:15:40,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-08T01:15:40,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5300 sec 2024-12-08T01:15:40,202 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620540201"}]},"ts":"1733620540201"} 2024-12-08T01:15:40,203 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T01:15:40,203 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T01:15:40,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5390 sec 2024-12-08T01:15:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T01:15:40,822 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:40,823 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T01:15:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,825 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-08T01:15:40,826 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,831 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:40,834 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/recovered.edits] 2024-12-08T01:15:40,839 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/d687640319c440998e2ff0d5b40d25f9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/d687640319c440998e2ff0d5b40d25f9 2024-12-08T01:15:40,841 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb30f400b64417a9f4641042efe1957 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/A/fbb30f400b64417a9f4641042efe1957 2024-12-08T01:15:40,845 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/e0eacb15bbc140f58dee3fd3796559e2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/e0eacb15bbc140f58dee3fd3796559e2 2024-12-08T01:15:40,847 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/f5d525b25d814ee48f5c6a427cced60f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/B/f5d525b25d814ee48f5c6a427cced60f 2024-12-08T01:15:40,850 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c262fec3a394492db10c6b729586fba5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/c262fec3a394492db10c6b729586fba5 2024-12-08T01:15:40,851 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f1199f6c72f94d11960df5b0d2f95ea9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/C/f1199f6c72f94d11960df5b0d2f95ea9 2024-12-08T01:15:40,853 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/recovered.edits/436.seqid to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49/recovered.edits/436.seqid 2024-12-08T01:15:40,853 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/836035c90977e14b643a4b4ed6128d49 2024-12-08T01:15:40,854 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T01:15:40,856 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,858 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T01:15:40,859 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T01:15:40,860 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,860 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T01:15:40,860 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733620540860"}]},"ts":"9223372036854775807"} 2024-12-08T01:15:40,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T01:15:40,862 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 836035c90977e14b643a4b4ed6128d49, NAME => 'TestAcidGuarantees,,1733620508736.836035c90977e14b643a4b4ed6128d49.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T01:15:40,862 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T01:15:40,862 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733620540862"}]},"ts":"9223372036854775807"} 2024-12-08T01:15:40,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T01:15:40,864 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-08T01:15:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-08T01:15:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T01:15:40,932 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:40,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 380) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9908 (was 9427) - AvailableMemoryMB LEAK? - 2024-12-08T01:15:40,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=9908 2024-12-08T01:15:40,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:15:40,958 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:15:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:40,959 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:15:40,960 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:40,960 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:15:40,960 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-12-08T01:15:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T01:15:40,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742157_1333 (size=1061) 2024-12-08T01:15:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T01:15:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T01:15:41,370 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:15:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742158_1334 (size=53) 2024-12-08T01:15:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T01:15:41,780 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:15:41,780 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c0ffe97318107908ae81381c359cc9b5, disabling compactions & flushes 2024-12-08T01:15:41,780 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:41,780 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:41,780 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. after waiting 0 ms 2024-12-08T01:15:41,780 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:41,780 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:41,780 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0ffe97318107908ae81381c359cc9b5: Waiting for close lock at 1733620541780Disabling compacts and flushes for region at 1733620541780Disabling writes for close at 1733620541780Writing region close event to WAL at 1733620541780Closed at 1733620541780 2024-12-08T01:15:41,782 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:15:41,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733620541782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620541782"}]},"ts":"1733620541782"} 2024-12-08T01:15:41,786 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:15:41,788 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:15:41,788 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620541788"}]},"ts":"1733620541788"} 2024-12-08T01:15:41,791 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T01:15:41,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, ASSIGN}] 2024-12-08T01:15:41,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, ASSIGN 2024-12-08T01:15:41,793 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, ASSIGN; state=OFFLINE, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=false 2024-12-08T01:15:41,944 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:41,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, ASSIGN because future has completed 2024-12-08T01:15:41,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630}] 2024-12-08T01:15:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T01:15:42,106 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:42,106 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:15:42,107 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,107 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:15:42,107 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,107 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,109 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,111 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:42,112 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0ffe97318107908ae81381c359cc9b5 columnFamilyName A 2024-12-08T01:15:42,112 DEBUG [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:42,113 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(327): Store=c0ffe97318107908ae81381c359cc9b5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:42,113 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,115 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:42,115 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0ffe97318107908ae81381c359cc9b5 columnFamilyName B 2024-12-08T01:15:42,115 DEBUG [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:42,116 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(327): Store=c0ffe97318107908ae81381c359cc9b5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:42,116 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,117 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:42,118 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0ffe97318107908ae81381c359cc9b5 columnFamilyName C 2024-12-08T01:15:42,118 DEBUG [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:42,119 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(327): Store=c0ffe97318107908ae81381c359cc9b5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:42,119 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:42,119 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,120 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,121 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,123 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,123 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,124 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:15:42,125 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,128 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:15:42,128 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened c0ffe97318107908ae81381c359cc9b5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67190812, jitterRate=0.0012211203575134277}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:15:42,129 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:42,129 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for c0ffe97318107908ae81381c359cc9b5: Running coprocessor pre-open hook at 1733620542107Writing region info on filesystem at 1733620542107Initializing all the Stores at 1733620542109 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620542109Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620542109Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620542109Cleaning up temporary data from old regions at 1733620542123 (+14 ms)Running coprocessor post-open hooks at 1733620542129 (+6 ms)Region opened successfully at 1733620542129 2024-12-08T01:15:42,130 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., pid=91, masterSystemTime=1733620542102 2024-12-08T01:15:42,132 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:42,132 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:42,133 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:42,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:15:42,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-08T01:15:42,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 in 186 msec 2024-12-08T01:15:42,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-08T01:15:42,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, ASSIGN in 344 msec 2024-12-08T01:15:42,138 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:15:42,138 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620542138"}]},"ts":"1733620542138"} 2024-12-08T01:15:42,139 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T01:15:42,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:15:42,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-12-08T01:15:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T01:15:43,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:43,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:43,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:43,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:43,114 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20edacad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:43,116 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df856c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:43,117 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:43,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:43,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:15:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:15:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ac18678 2024-12-08T01:15:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:15:43,122 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:15:43,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:15:43,123 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:15:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T01:15:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742159_1335 (size=1097) 2024-12-08T01:15:43,535 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T01:15:43,535 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T01:15:43,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:15:43,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, REOPEN/MOVE}] 2024-12-08T01:15:43,539 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, REOPEN/MOVE 2024-12-08T01:15:43,540 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:43,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, REOPEN/MOVE because future has completed 2024-12-08T01:15:43,542 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:15:43,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630}] 2024-12-08T01:15:43,694 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:43,694 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:15:43,694 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing c0ffe97318107908ae81381c359cc9b5, disabling compactions & flushes 2024-12-08T01:15:43,694 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:43,694 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:43,694 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. after waiting 0 ms 2024-12-08T01:15:43,694 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:43,698 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T01:15:43,699 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:43,699 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for c0ffe97318107908ae81381c359cc9b5: Waiting for close lock at 1733620543694Running coprocessor pre-close hooks at 1733620543694Disabling compacts and flushes for region at 1733620543694Disabling writes for close at 1733620543694Writing region close event to WAL at 1733620543695 (+1 ms)Running coprocessor post-close hooks at 1733620543699 (+4 ms)Closed at 1733620543699 2024-12-08T01:15:43,699 WARN [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: c0ffe97318107908ae81381c359cc9b5 to self. 2024-12-08T01:15:43,701 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:43,702 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=CLOSED 2024-12-08T01:15:43,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:15:43,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-08T01:15:43,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 in 162 msec 2024-12-08T01:15:43,707 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, REOPEN/MOVE; state=CLOSED, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=true 2024-12-08T01:15:43,858 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:43,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, REOPEN/MOVE because future has completed 2024-12-08T01:15:43,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630}] 2024-12-08T01:15:44,015 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,015 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:15:44,016 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,016 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:15:44,016 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,016 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,021 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,022 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:44,023 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0ffe97318107908ae81381c359cc9b5 columnFamilyName A 2024-12-08T01:15:44,024 DEBUG [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:44,025 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(327): Store=c0ffe97318107908ae81381c359cc9b5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:44,025 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,026 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:44,026 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0ffe97318107908ae81381c359cc9b5 columnFamilyName B 2024-12-08T01:15:44,026 DEBUG [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:44,026 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(327): Store=c0ffe97318107908ae81381c359cc9b5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:44,027 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,027 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:15:44,027 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0ffe97318107908ae81381c359cc9b5 columnFamilyName C 2024-12-08T01:15:44,027 DEBUG [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:44,028 INFO [StoreOpener-c0ffe97318107908ae81381c359cc9b5-1 {}] regionserver.HStore(327): Store=c0ffe97318107908ae81381c359cc9b5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:15:44,028 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,028 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,028 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,029 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,030 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,030 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,031 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:15:44,032 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,033 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened c0ffe97318107908ae81381c359cc9b5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69252460, jitterRate=0.03194206953048706}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:15:44,033 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,033 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for c0ffe97318107908ae81381c359cc9b5: Running coprocessor pre-open hook at 1733620544016Writing region info on filesystem at 1733620544016Initializing all the Stores at 1733620544021 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620544021Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620544021Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620544021Cleaning up temporary data from old regions at 1733620544030 (+9 ms)Running coprocessor post-open hooks at 1733620544033 (+3 ms)Region opened successfully at 1733620544033 2024-12-08T01:15:44,034 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., pid=96, masterSystemTime=1733620544012 2024-12-08T01:15:44,036 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,036 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,037 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=OPEN, openSeqNum=5, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:15:44,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:15:44,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-12-08T01:15:44,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 in 180 msec 2024-12-08T01:15:44,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-08T01:15:44,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, REOPEN/MOVE in 503 msec 2024-12-08T01:15:44,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-08T01:15:44,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-12-08T01:15:44,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-12-08T01:15:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-08T01:15:44,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:44,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,054 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4882bb07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,056 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459611cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,058 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae2ae06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f4ce7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,063 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3ee3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50121f9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,070 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b147a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,073 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fba22b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,077 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,078 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16dfc3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,080 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6bb987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,083 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692b5971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,085 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce044c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,088 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ce8d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,090 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c277aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209a0da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d66261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52cc992b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a4ca7c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:15:44,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:15:44,114 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:15:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:15:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:15:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71710084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:15:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:15:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:15:44,117 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:15:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f21b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:15:44,120 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,122 DEBUG [Thread-1537 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,122 DEBUG [Thread-1537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,130 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,130 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,132 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,134 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,145 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,145 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-08T01:15:44,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,148 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:44,149 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:15:44,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:44,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:44,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:44,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:44,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:44,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:44,161 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,180 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,182 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,183 DEBUG [Thread-1543 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,184 DEBUG [Thread-1543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,184 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:15:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:15:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:15:44,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:15:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:15:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620604187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620604190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620604192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620604193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620604194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120859366b9493174bad95a0052b2533efee_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620544140/Put/seqid=0 2024-12-08T01:15:44,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742160_1336 (size=17034) 2024-12-08T01:15:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:44,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:44,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:44,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620604304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620604312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620604312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620604312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620604312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:44,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620604512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620604532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620604532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620604532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620604533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:44,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:44,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:44,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120859366b9493174bad95a0052b2533efee_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120859366b9493174bad95a0052b2533efee_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:44,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/11395b55fe7b45c6ae8e625be886dd97, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:44,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/11395b55fe7b45c6ae8e625be886dd97 is 175, key is test_row_0/A:col10/1733620544140/Put/seqid=0 2024-12-08T01:15:44,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742161_1337 (size=48139) 2024-12-08T01:15:44,656 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/11395b55fe7b45c6ae8e625be886dd97 2024-12-08T01:15:44,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/4dde575a406046bcba081c102f07b3fd is 50, key is test_row_0/B:col10/1733620544140/Put/seqid=0 2024-12-08T01:15:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742162_1338 (size=12001) 2024-12-08T01:15:44,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:44,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:44,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620604821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620604852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620604852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620604852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620604852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:44,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:44,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/4dde575a406046bcba081c102f07b3fd 2024-12-08T01:15:45,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/dbe1763ac4a14bb495e33fdcf5a92b55 is 50, key is test_row_0/C:col10/1733620544140/Put/seqid=0 2024-12-08T01:15:45,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742163_1339 (size=12001) 2024-12-08T01:15:45,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620605332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620605362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620605362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620605362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620605362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:45,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:45,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:45,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:45,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:45,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:45,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/dbe1763ac4a14bb495e33fdcf5a92b55 2024-12-08T01:15:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/11395b55fe7b45c6ae8e625be886dd97 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97 2024-12-08T01:15:45,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97, entries=250, sequenceid=16, filesize=47.0 K 2024-12-08T01:15:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/4dde575a406046bcba081c102f07b3fd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4dde575a406046bcba081c102f07b3fd 2024-12-08T01:15:45,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4dde575a406046bcba081c102f07b3fd, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T01:15:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/dbe1763ac4a14bb495e33fdcf5a92b55 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/dbe1763ac4a14bb495e33fdcf5a92b55 2024-12-08T01:15:45,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/dbe1763ac4a14bb495e33fdcf5a92b55, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T01:15:45,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c0ffe97318107908ae81381c359cc9b5 in 1470ms, sequenceid=16, compaction requested=false 2024-12-08T01:15:45,621 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T01:15:45,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:45,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:45,683 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:45,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b13abc1ff05f4e27822907f973772038_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620544191/Put/seqid=0 2024-12-08T01:15:45,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742164_1340 (size=12154) 2024-12-08T01:15:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:45,698 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b13abc1ff05f4e27822907f973772038_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b13abc1ff05f4e27822907f973772038_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:45,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/6036c6f23f1f43d681d61d4d6138f87c, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:45,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/6036c6f23f1f43d681d61d4d6138f87c is 175, key is test_row_0/A:col10/1733620544191/Put/seqid=0 2024-12-08T01:15:45,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742165_1341 (size=30955) 2024-12-08T01:15:45,723 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:15:46,109 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/6036c6f23f1f43d681d61d4d6138f87c 2024-12-08T01:15:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/716dfbbb495a49fcb43fc65adfd44007 is 50, key is test_row_0/B:col10/1733620544191/Put/seqid=0 2024-12-08T01:15:46,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742166_1342 (size=12001) 2024-12-08T01:15:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:46,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620606382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620606382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620606384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,393 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in c0ffe97318107908ae81381c359cc9b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620606385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620606385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620606502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620606502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620606503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620606503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,525 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/716dfbbb495a49fcb43fc65adfd44007 2024-12-08T01:15:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/31a7561deecd4d2c9038b6992e50ca6b is 50, key is test_row_0/C:col10/1733620544191/Put/seqid=0 2024-12-08T01:15:46,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742167_1343 (size=12001) 2024-12-08T01:15:46,559 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/31a7561deecd4d2c9038b6992e50ca6b 2024-12-08T01:15:46,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/6036c6f23f1f43d681d61d4d6138f87c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c 2024-12-08T01:15:46,575 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c, entries=150, sequenceid=36, filesize=30.2 K 2024-12-08T01:15:46,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/716dfbbb495a49fcb43fc65adfd44007 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/716dfbbb495a49fcb43fc65adfd44007 2024-12-08T01:15:46,582 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/716dfbbb495a49fcb43fc65adfd44007, entries=150, sequenceid=36, filesize=11.7 K 2024-12-08T01:15:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/31a7561deecd4d2c9038b6992e50ca6b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/31a7561deecd4d2c9038b6992e50ca6b 2024-12-08T01:15:46,627 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/31a7561deecd4d2c9038b6992e50ca6b, entries=150, sequenceid=36, filesize=11.7 K 2024-12-08T01:15:46,628 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c0ffe97318107908ae81381c359cc9b5 in 945ms, sequenceid=36, compaction requested=false 2024-12-08T01:15:46,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:46,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:46,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-08T01:15:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-08T01:15:46,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-08T01:15:46,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4800 sec 2024-12-08T01:15:46,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.4850 sec 2024-12-08T01:15:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:46,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:46,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:46,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085abf9b5d4f994c47af956846a7385f1c_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620606750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620606759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620606760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620606763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742168_1344 (size=14594) 2024-12-08T01:15:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620606871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620606882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620606882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620606882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620607092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620607092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620607092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620607092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:47,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085abf9b5d4f994c47af956846a7385f1c_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085abf9b5d4f994c47af956846a7385f1c_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:47,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/df5d7a4e1d3f4d7e9573f2de89ca037d, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:47,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/df5d7a4e1d3f4d7e9573f2de89ca037d is 175, key is test_row_0/A:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742169_1345 (size=39549) 2024-12-08T01:15:47,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620607402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620607403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620607403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620607403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/df5d7a4e1d3f4d7e9573f2de89ca037d 2024-12-08T01:15:47,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/7248a1d8ffc34828a303b6badac81921 is 50, key is test_row_0/B:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:47,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742170_1346 (size=12001) 2024-12-08T01:15:47,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620607912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620607912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620607912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620607912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:48,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/7248a1d8ffc34828a303b6badac81921 2024-12-08T01:15:48,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/c22a0f23078645568f7c3f6aa77e5aaf is 50, key is test_row_0/C:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:48,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742171_1347 (size=12001) 2024-12-08T01:15:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T01:15:48,312 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:48,313 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-08T01:15:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T01:15:48,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:48,315 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:48,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:48,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/c22a0f23078645568f7c3f6aa77e5aaf 2024-12-08T01:15:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620608412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:48,417 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in c0ffe97318107908ae81381c359cc9b5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:48,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/df5d7a4e1d3f4d7e9573f2de89ca037d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d 2024-12-08T01:15:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T01:15:48,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d, entries=200, sequenceid=50, filesize=38.6 K 2024-12-08T01:15:48,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/7248a1d8ffc34828a303b6badac81921 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7248a1d8ffc34828a303b6badac81921 2024-12-08T01:15:48,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7248a1d8ffc34828a303b6badac81921, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:15:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/c22a0f23078645568f7c3f6aa77e5aaf as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/c22a0f23078645568f7c3f6aa77e5aaf 2024-12-08T01:15:48,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/c22a0f23078645568f7c3f6aa77e5aaf, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:15:48,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c0ffe97318107908ae81381c359cc9b5 in 1718ms, sequenceid=50, compaction requested=true 2024-12-08T01:15:48,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:48,434 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:48,434 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:48,435 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:48,435 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:48,435 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/A is initiating minor compaction (all files) 2024-12-08T01:15:48,435 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/B is initiating minor compaction (all files) 2024-12-08T01:15:48,435 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/B in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:48,435 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/A in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:48,435 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4dde575a406046bcba081c102f07b3fd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/716dfbbb495a49fcb43fc65adfd44007, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7248a1d8ffc34828a303b6badac81921] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.2 K 2024-12-08T01:15:48,435 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=115.9 K 2024-12-08T01:15:48,435 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:48,435 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d] 2024-12-08T01:15:48,437 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11395b55fe7b45c6ae8e625be886dd97, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733620544140 2024-12-08T01:15:48,437 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dde575a406046bcba081c102f07b3fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733620544140 2024-12-08T01:15:48,437 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 716dfbbb495a49fcb43fc65adfd44007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733620544182 2024-12-08T01:15:48,437 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6036c6f23f1f43d681d61d4d6138f87c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733620544182 2024-12-08T01:15:48,437 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7248a1d8ffc34828a303b6badac81921, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620546379 2024-12-08T01:15:48,437 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5d7a4e1d3f4d7e9573f2de89ca037d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620546379 2024-12-08T01:15:48,444 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:48,444 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/379fd2c0bda1467aa779d6d925437fb2 is 50, key is test_row_0/B:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:48,445 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:48,449 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208b7efffe80fd140949d0fb1f227d08944_c0ffe97318107908ae81381c359cc9b5 store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:48,451 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208b7efffe80fd140949d0fb1f227d08944_c0ffe97318107908ae81381c359cc9b5, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:48,451 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7efffe80fd140949d0fb1f227d08944_c0ffe97318107908ae81381c359cc9b5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:48,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742172_1348 (size=12104) 2024-12-08T01:15:48,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742173_1349 (size=4469) 2024-12-08T01:15:48,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T01:15:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:48,467 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:15:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b3931695d7d4912831838d1e7a4c461_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620546761/Put/seqid=0 2024-12-08T01:15:48,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742174_1350 (size=12154) 2024-12-08T01:15:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T01:15:48,861 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/379fd2c0bda1467aa779d6d925437fb2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/379fd2c0bda1467aa779d6d925437fb2 2024-12-08T01:15:48,864 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:48,864 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/ff0cd59264a149c88b1eda6640346346 is 175, key is test_row_0/A:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:48,869 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/B of c0ffe97318107908ae81381c359cc9b5 into 379fd2c0bda1467aa779d6d925437fb2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:48,869 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:48,869 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/B, priority=13, startTime=1733620548434; duration=0sec 2024-12-08T01:15:48,869 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:48,869 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:B 2024-12-08T01:15:48,869 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742175_1351 (size=31058) 2024-12-08T01:15:48,876 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:48,876 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/C is initiating minor compaction (all files) 2024-12-08T01:15:48,876 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/C in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:48,876 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/dbe1763ac4a14bb495e33fdcf5a92b55, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/31a7561deecd4d2c9038b6992e50ca6b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/c22a0f23078645568f7c3f6aa77e5aaf] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.2 K 2024-12-08T01:15:48,877 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting dbe1763ac4a14bb495e33fdcf5a92b55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733620544140 2024-12-08T01:15:48,889 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a7561deecd4d2c9038b6992e50ca6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733620544182 2024-12-08T01:15:48,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:48,890 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c22a0f23078645568f7c3f6aa77e5aaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620546379 2024-12-08T01:15:48,895 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b3931695d7d4912831838d1e7a4c461_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087b3931695d7d4912831838d1e7a4c461_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:48,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/f9d94484b1b64ac2b127ac1d8fc0fe11, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:48,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/f9d94484b1b64ac2b127ac1d8fc0fe11 is 175, key is test_row_0/A:col10/1733620546761/Put/seqid=0 2024-12-08T01:15:48,905 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:48,905 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/fcd4af1afc33402d8841e5ad86ca2547 is 50, key is test_row_0/C:col10/1733620546381/Put/seqid=0 2024-12-08T01:15:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742176_1352 (size=30955) 2024-12-08T01:15:48,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T01:15:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742177_1353 (size=12104) 2024-12-08T01:15:48,954 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/fcd4af1afc33402d8841e5ad86ca2547 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fcd4af1afc33402d8841e5ad86ca2547 2024-12-08T01:15:48,965 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/C of c0ffe97318107908ae81381c359cc9b5 into fcd4af1afc33402d8841e5ad86ca2547(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:48,965 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:48,965 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/C, priority=13, startTime=1733620548434; duration=0sec 2024-12-08T01:15:48,966 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:48,966 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:C 2024-12-08T01:15:48,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620608961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:48,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620608961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620608969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:48,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620608973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620609081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620609081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620609095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620609095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,282 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/ff0cd59264a149c88b1eda6640346346 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/ff0cd59264a149c88b1eda6640346346 2024-12-08T01:15:49,287 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/A of c0ffe97318107908ae81381c359cc9b5 into ff0cd59264a149c88b1eda6640346346(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:49,288 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:49,288 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/A, priority=13, startTime=1733620548433; duration=0sec 2024-12-08T01:15:49,288 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:49,288 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:A 2024-12-08T01:15:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620609292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620609292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620609311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620609312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,329 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/f9d94484b1b64ac2b127ac1d8fc0fe11 2024-12-08T01:15:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/7e09c6691b7a451dbaeb706592f9f133 is 50, key is test_row_0/B:col10/1733620546761/Put/seqid=0 2024-12-08T01:15:49,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742178_1354 (size=12001) 2024-12-08T01:15:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T01:15:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620609602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620609612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620609622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620609622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:49,746 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/7e09c6691b7a451dbaeb706592f9f133 2024-12-08T01:15:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/fc99d67e80dd49148f9ae8c0ccbc2cc8 is 50, key is test_row_0/C:col10/1733620546761/Put/seqid=0 2024-12-08T01:15:49,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742179_1355 (size=12001) 2024-12-08T01:15:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620610112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:50,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620610122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620610132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620610132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:50,158 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/fc99d67e80dd49148f9ae8c0ccbc2cc8 2024-12-08T01:15:50,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/f9d94484b1b64ac2b127ac1d8fc0fe11 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11 2024-12-08T01:15:50,165 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11, entries=150, sequenceid=68, filesize=30.2 K 2024-12-08T01:15:50,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/7e09c6691b7a451dbaeb706592f9f133 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7e09c6691b7a451dbaeb706592f9f133 2024-12-08T01:15:50,170 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7e09c6691b7a451dbaeb706592f9f133, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T01:15:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/fc99d67e80dd49148f9ae8c0ccbc2cc8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fc99d67e80dd49148f9ae8c0ccbc2cc8 2024-12-08T01:15:50,175 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fc99d67e80dd49148f9ae8c0ccbc2cc8, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T01:15:50,176 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0ffe97318107908ae81381c359cc9b5 in 1709ms, sequenceid=68, compaction requested=false 2024-12-08T01:15:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-08T01:15:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-08T01:15:50,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-08T01:15:50,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-12-08T01:15:50,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8670 sec 2024-12-08T01:15:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T01:15:50,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:50,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-08T01:15:50,464 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T01:15:50,465 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:50,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:50,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T01:15:50,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T01:15:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:50,617 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:15:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:50,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f7b245bea8cd4d0bbb685575165e9a51_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620548970/Put/seqid=0 2024-12-08T01:15:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742180_1356 (size=12154) 2024-12-08T01:15:50,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:50,646 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f7b245bea8cd4d0bbb685575165e9a51_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f7b245bea8cd4d0bbb685575165e9a51_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/e30328b20d7b429a8ab0002c70697716, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:50,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/e30328b20d7b429a8ab0002c70697716 is 175, key is test_row_0/A:col10/1733620548970/Put/seqid=0 2024-12-08T01:15:50,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742181_1357 (size=30955) 2024-12-08T01:15:50,658 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/e30328b20d7b429a8ab0002c70697716 2024-12-08T01:15:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/57d34d13a3954f6d97d3a2b952defb9a is 50, key is test_row_0/B:col10/1733620548970/Put/seqid=0 2024-12-08T01:15:50,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742182_1358 (size=12001) 2024-12-08T01:15:50,684 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/57d34d13a3954f6d97d3a2b952defb9a 2024-12-08T01:15:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0 is 50, key is test_row_0/C:col10/1733620548970/Put/seqid=0 2024-12-08T01:15:50,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742183_1359 (size=12001) 2024-12-08T01:15:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T01:15:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T01:15:51,112 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0 2024-12-08T01:15:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/e30328b20d7b429a8ab0002c70697716 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716 2024-12-08T01:15:51,121 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716, entries=150, sequenceid=85, filesize=30.2 K 2024-12-08T01:15:51,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/57d34d13a3954f6d97d3a2b952defb9a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/57d34d13a3954f6d97d3a2b952defb9a 2024-12-08T01:15:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,125 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/57d34d13a3954f6d97d3a2b952defb9a, entries=150, sequenceid=85, filesize=11.7 K 2024-12-08T01:15:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0 2024-12-08T01:15:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,131 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0, entries=150, sequenceid=85, filesize=11.7 K 2024-12-08T01:15:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,132 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for c0ffe97318107908ae81381c359cc9b5 in 515ms, sequenceid=85, compaction requested=true 2024-12-08T01:15:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-08T01:15:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-08T01:15:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-08T01:15:51,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 669 msec 2024-12-08T01:15:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 676 msec 2024-12-08T01:15:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:51,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:15:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120817be125e942840c6ae826331ae56d1cd_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742184_1360 (size=21918) 2024-12-08T01:15:51,292 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:51,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120817be125e942840c6ae826331ae56d1cd_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120817be125e942840c6ae826331ae56d1cd_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:51,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/c7f513c11a994feab7aa10c4a3d0c8cd, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:51,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/c7f513c11a994feab7aa10c4a3d0c8cd is 175, key is test_row_0/A:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:51,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742185_1361 (size=65323) 2024-12-08T01:15:51,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620551309, max timestamp=1733620551331Num uniques 859; 2024-12-08T01:15:51,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620551309, max timestamp=1733620551331Num uniques 859; 2024-12-08T01:15:51,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620551309, max timestamp=1733620551331Num uniques 859; 2024-12-08T01:15:51,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:15:51,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:15:51,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:15:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620611366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620611367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620611378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620611378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620611481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620611482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620611492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620611492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T01:15:51,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:51,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-08T01:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:51,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:51,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:51,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:51,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620611691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620611692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620611702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620611702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:51,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/c7f513c11a994feab7aa10c4a3d0c8cd 2024-12-08T01:15:51,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/0e85399350ed4fe59ed15bfba7e97717 is 50, key is test_row_0/B:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:51,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742186_1362 (size=12001) 2024-12-08T01:15:51,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:51,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:51,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:51,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:51,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620612001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620612012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620612022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620612022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/0e85399350ed4fe59ed15bfba7e97717 2024-12-08T01:15:52,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/4d38ac6855634c31a7f2d93fbd6d1977 is 50, key is test_row_0/C:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:52,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742187_1363 (size=12001) 2024-12-08T01:15:52,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:52,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:52,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:52,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620612452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,457 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in c0ffe97318107908ae81381c359cc9b5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:15:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620612512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620612522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620612532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620612532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:52,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/4d38ac6855634c31a7f2d93fbd6d1977 2024-12-08T01:15:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/c7f513c11a994feab7aa10c4a3d0c8cd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd 2024-12-08T01:15:52,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd, entries=350, sequenceid=95, filesize=63.8 K 2024-12-08T01:15:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/0e85399350ed4fe59ed15bfba7e97717 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/0e85399350ed4fe59ed15bfba7e97717 2024-12-08T01:15:52,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/0e85399350ed4fe59ed15bfba7e97717, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T01:15:52,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/4d38ac6855634c31a7f2d93fbd6d1977 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4d38ac6855634c31a7f2d93fbd6d1977 2024-12-08T01:15:52,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4d38ac6855634c31a7f2d93fbd6d1977, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T01:15:52,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c0ffe97318107908ae81381c359cc9b5 in 1312ms, sequenceid=95, compaction requested=true 2024-12-08T01:15:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:52,571 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:52,571 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:52,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:52,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:52,572 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:52,572 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158291 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:52,572 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/B is initiating minor compaction (all files) 2024-12-08T01:15:52,572 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/A is initiating minor compaction (all files) 2024-12-08T01:15:52,572 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/B in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,572 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/A in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,572 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/379fd2c0bda1467aa779d6d925437fb2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7e09c6691b7a451dbaeb706592f9f133, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/57d34d13a3954f6d97d3a2b952defb9a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/0e85399350ed4fe59ed15bfba7e97717] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=47.0 K 2024-12-08T01:15:52,573 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/ff0cd59264a149c88b1eda6640346346, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=154.6 K 2024-12-08T01:15:52,573 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/ff0cd59264a149c88b1eda6640346346, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd] 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 379fd2c0bda1467aa779d6d925437fb2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620546379 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff0cd59264a149c88b1eda6640346346, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620546379 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e09c6691b7a451dbaeb706592f9f133, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733620546744 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d94484b1b64ac2b127ac1d8fc0fe11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733620546744 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d34d13a3954f6d97d3a2b952defb9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620548957 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e30328b20d7b429a8ab0002c70697716, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620548957 2024-12-08T01:15:52,573 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e85399350ed4fe59ed15bfba7e97717, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620551208 2024-12-08T01:15:52,574 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7f513c11a994feab7aa10c4a3d0c8cd, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620551205 2024-12-08T01:15:52,582 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:52,583 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:52,584 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/697d5cc6a9ae45a1b609251a3ab90275 is 50, key is test_row_0/B:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:52,586 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208408a425c2d804491b672ca0af7259f8f_c0ffe97318107908ae81381c359cc9b5 store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:52,588 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208408a425c2d804491b672ca0af7259f8f_c0ffe97318107908ae81381c359cc9b5, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:52,588 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208408a425c2d804491b672ca0af7259f8f_c0ffe97318107908ae81381c359cc9b5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742189_1365 (size=4469) 2024-12-08T01:15:52,594 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#A#compaction#313 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:52,594 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/99f8da3058d34f22bf3e9c1abe38af2c is 175, key is test_row_0/A:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:52,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742188_1364 (size=12241) 2024-12-08T01:15:52,604 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/697d5cc6a9ae45a1b609251a3ab90275 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/697d5cc6a9ae45a1b609251a3ab90275 2024-12-08T01:15:52,607 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/B of c0ffe97318107908ae81381c359cc9b5 into 697d5cc6a9ae45a1b609251a3ab90275(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:52,607 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:52,607 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/B, priority=12, startTime=1733620552571; duration=0sec 2024-12-08T01:15:52,607 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:52,607 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:B 2024-12-08T01:15:52,608 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:15:52,608 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:15:52,608 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/C is initiating minor compaction (all files) 2024-12-08T01:15:52,609 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/C in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,609 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fcd4af1afc33402d8841e5ad86ca2547, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fc99d67e80dd49148f9ae8c0ccbc2cc8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4d38ac6855634c31a7f2d93fbd6d1977] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=47.0 K 2024-12-08T01:15:52,609 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fcd4af1afc33402d8841e5ad86ca2547, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620546379 2024-12-08T01:15:52,613 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fc99d67e80dd49148f9ae8c0ccbc2cc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733620546744 2024-12-08T01:15:52,613 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbf42c0c2ea4bdc949a88a8cd79e2a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620548957 2024-12-08T01:15:52,613 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d38ac6855634c31a7f2d93fbd6d1977, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620551208 2024-12-08T01:15:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742190_1366 (size=31195) 2024-12-08T01:15:52,619 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:52,620 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/36ba888bfc254308b4a4f15d516f58fd is 50, key is test_row_0/C:col10/1733620551208/Put/seqid=0 2024-12-08T01:15:52,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742191_1367 (size=12241) 2024-12-08T01:15:52,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T01:15:52,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:52,675 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T01:15:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:15:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:15:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:15:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f724fb1f994a4421a125e881f7693a1e_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620551376/Put/seqid=0 2024-12-08T01:15:52,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742192_1368 (size=12254) 2024-12-08T01:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:53,018 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/99f8da3058d34f22bf3e9c1abe38af2c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/99f8da3058d34f22bf3e9c1abe38af2c 2024-12-08T01:15:53,021 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/A of c0ffe97318107908ae81381c359cc9b5 into 99f8da3058d34f22bf3e9c1abe38af2c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:53,021 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:53,021 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/A, priority=12, startTime=1733620552571; duration=0sec 2024-12-08T01:15:53,021 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:53,021 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:A 2024-12-08T01:15:53,026 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/36ba888bfc254308b4a4f15d516f58fd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/36ba888bfc254308b4a4f15d516f58fd 2024-12-08T01:15:53,030 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/C of c0ffe97318107908ae81381c359cc9b5 into 36ba888bfc254308b4a4f15d516f58fd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:53,030 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:53,030 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/C, priority=12, startTime=1733620552571; duration=0sec 2024-12-08T01:15:53,030 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:53,030 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:C 2024-12-08T01:15:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:53,094 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f724fb1f994a4421a125e881f7693a1e_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f724fb1f994a4421a125e881f7693a1e_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/7115cdc39dbb4673a94799b93eeb3eec, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/7115cdc39dbb4673a94799b93eeb3eec is 175, key is test_row_0/A:col10/1733620551376/Put/seqid=0 2024-12-08T01:15:53,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742193_1369 (size=31055) 2024-12-08T01:15:53,500 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/7115cdc39dbb4673a94799b93eeb3eec 2024-12-08T01:15:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/74f4ef151d6c47a3b164468dbe30e14b is 50, key is test_row_0/B:col10/1733620551376/Put/seqid=0 2024-12-08T01:15:53,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742194_1370 (size=12101) 2024-12-08T01:15:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:53,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:53,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620613553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620613559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620613560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620613560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620613672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620613672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620613672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620613672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620613881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620613881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620613882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620613882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:53,910 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/74f4ef151d6c47a3b164468dbe30e14b 2024-12-08T01:15:53,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5 is 50, key is test_row_0/C:col10/1733620551376/Put/seqid=0 2024-12-08T01:15:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742195_1371 (size=12101) 2024-12-08T01:15:54,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620614202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620614202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620614202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620614202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,323 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5 2024-12-08T01:15:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/7115cdc39dbb4673a94799b93eeb3eec as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec 2024-12-08T01:15:54,330 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec, entries=150, sequenceid=132, filesize=30.3 K 2024-12-08T01:15:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/74f4ef151d6c47a3b164468dbe30e14b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/74f4ef151d6c47a3b164468dbe30e14b 2024-12-08T01:15:54,333 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/74f4ef151d6c47a3b164468dbe30e14b, entries=150, sequenceid=132, filesize=11.8 K 2024-12-08T01:15:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5 2024-12-08T01:15:54,338 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5, entries=150, sequenceid=132, filesize=11.8 K 2024-12-08T01:15:54,339 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for c0ffe97318107908ae81381c359cc9b5 in 1664ms, sequenceid=132, compaction requested=false 2024-12-08T01:15:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-08T01:15:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-08T01:15:54,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-08T01:15:54,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7360 sec 2024-12-08T01:15:54,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7400 sec 2024-12-08T01:15:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:54,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T01:15:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:54,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ea9dc5f2d38e4a5bb044e30339482d0c_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:54,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742196_1372 (size=14794) 2024-12-08T01:15:54,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620614770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620614770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620614778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620614778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620614882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620614882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620614892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620614892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:54,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:15:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620615092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620615092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620615102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620615102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:55,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ea9dc5f2d38e4a5bb044e30339482d0c_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ea9dc5f2d38e4a5bb044e30339482d0c_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:55,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/471283bab20c432d812d8288973b055a, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:55,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/471283bab20c432d812d8288973b055a is 175, key is test_row_0/A:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:55,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742197_1373 (size=39749) 2024-12-08T01:15:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620615401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620615401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620615412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620615412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/471283bab20c432d812d8288973b055a 2024-12-08T01:15:55,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/4a126f3a840147aa94a1aba58e29d4a1 is 50, key is test_row_0/B:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:55,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742198_1374 (size=12151) 2024-12-08T01:15:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T01:15:55,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:55,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-08T01:15:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:55,764 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:55,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:55,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:55,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T01:15:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:55,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:55,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620615922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620615922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620615931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620615932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:55,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/4a126f3a840147aa94a1aba58e29d4a1 2024-12-08T01:15:55,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/7ee12f0048dc47edaa3683aae2e8dfc0 is 50, key is test_row_0/C:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:55,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742199_1375 (size=12151) 2024-12-08T01:15:56,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T01:15:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:56,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T01:15:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/7ee12f0048dc47edaa3683aae2e8dfc0 2024-12-08T01:15:56,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/471283bab20c432d812d8288973b055a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a 2024-12-08T01:15:56,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a, entries=200, sequenceid=146, filesize=38.8 K 2024-12-08T01:15:56,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/4a126f3a840147aa94a1aba58e29d4a1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4a126f3a840147aa94a1aba58e29d4a1 2024-12-08T01:15:56,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4a126f3a840147aa94a1aba58e29d4a1, entries=150, sequenceid=146, filesize=11.9 K 2024-12-08T01:15:56,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/7ee12f0048dc47edaa3683aae2e8dfc0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7ee12f0048dc47edaa3683aae2e8dfc0 2024-12-08T01:15:56,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T01:15:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:15:56,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7ee12f0048dc47edaa3683aae2e8dfc0, entries=150, sequenceid=146, filesize=11.9 K 2024-12-08T01:15:56,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c0ffe97318107908ae81381c359cc9b5 in 1652ms, sequenceid=146, compaction requested=true 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:56,379 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:15:56,379 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:15:56,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:56,380 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:56,380 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/A is initiating minor compaction (all files) 2024-12-08T01:15:56,380 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/A in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,380 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/99f8da3058d34f22bf3e9c1abe38af2c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=99.6 K 2024-12-08T01:15:56,380 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,380 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/99f8da3058d34f22bf3e9c1abe38af2c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a] 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/B is initiating minor compaction (all files) 2024-12-08T01:15:56,381 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/B in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,381 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/697d5cc6a9ae45a1b609251a3ab90275, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/74f4ef151d6c47a3b164468dbe30e14b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4a126f3a840147aa94a1aba58e29d4a1] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.6 K 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99f8da3058d34f22bf3e9c1abe38af2c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620551208 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 697d5cc6a9ae45a1b609251a3ab90275, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620551208 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7115cdc39dbb4673a94799b93eeb3eec, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733620551365 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f4ef151d6c47a3b164468dbe30e14b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733620551365 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471283bab20c432d812d8288973b055a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733620553552 2024-12-08T01:15:56,381 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a126f3a840147aa94a1aba58e29d4a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733620553552 2024-12-08T01:15:56,390 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:56,391 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/739b054ba6e94f22a5dec764f4dd73d6 is 50, key is test_row_0/B:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:56,391 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:56,395 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412081eeb7d48fc8e4589a81eb98530e226c8_c0ffe97318107908ae81381c359cc9b5 store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:56,396 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412081eeb7d48fc8e4589a81eb98530e226c8_c0ffe97318107908ae81381c359cc9b5, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:56,397 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081eeb7d48fc8e4589a81eb98530e226c8_c0ffe97318107908ae81381c359cc9b5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742200_1376 (size=12493) 2024-12-08T01:15:56,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742201_1377 (size=4469) 2024-12-08T01:15:56,403 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#A#compaction#322 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:56,403 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/4e70e11984024164a2a568289687a396 is 175, key is test_row_0/A:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:56,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742202_1378 (size=31447) 2024-12-08T01:15:56,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T01:15:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,528 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:15:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:56,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083233a40390ec4accb160b34f48971c3c_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620554776/Put/seqid=0 2024-12-08T01:15:56,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742203_1379 (size=12304) 2024-12-08T01:15:56,803 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/739b054ba6e94f22a5dec764f4dd73d6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/739b054ba6e94f22a5dec764f4dd73d6 2024-12-08T01:15:56,807 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/B of c0ffe97318107908ae81381c359cc9b5 into 739b054ba6e94f22a5dec764f4dd73d6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:56,808 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:56,808 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/B, priority=13, startTime=1733620556379; duration=0sec 2024-12-08T01:15:56,808 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:15:56,808 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:B 2024-12-08T01:15:56,808 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:15:56,810 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:15:56,810 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/C is initiating minor compaction (all files) 2024-12-08T01:15:56,810 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/C in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:56,810 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/36ba888bfc254308b4a4f15d516f58fd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7ee12f0048dc47edaa3683aae2e8dfc0] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.6 K 2024-12-08T01:15:56,812 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ba888bfc254308b4a4f15d516f58fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620551208 2024-12-08T01:15:56,815 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/4e70e11984024164a2a568289687a396 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/4e70e11984024164a2a568289687a396 2024-12-08T01:15:56,816 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7f2d8fc03d4e33a1500d33ab3fbaf5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733620551365 2024-12-08T01:15:56,816 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee12f0048dc47edaa3683aae2e8dfc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733620553552 2024-12-08T01:15:56,818 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/A of c0ffe97318107908ae81381c359cc9b5 into 4e70e11984024164a2a568289687a396(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:56,818 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:56,818 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/A, priority=13, startTime=1733620556379; duration=0sec 2024-12-08T01:15:56,818 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:56,818 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:A 2024-12-08T01:15:56,824 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:15:56,824 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/79d6947af0ca4963a66550c95b7d7a55 is 50, key is test_row_0/C:col10/1733620553559/Put/seqid=0 2024-12-08T01:15:56,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742204_1380 (size=12493) 2024-12-08T01:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:56,945 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083233a40390ec4accb160b34f48971c3c_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083233a40390ec4accb160b34f48971c3c_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:56,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:56,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:56,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046 is 175, key is test_row_0/A:col10/1733620554776/Put/seqid=0 2024-12-08T01:15:56,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742205_1381 (size=31105) 2024-12-08T01:15:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620616956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620616962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620616963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620616964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620617072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620617072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620617082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620617082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,231 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/79d6947af0ca4963a66550c95b7d7a55 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/79d6947af0ca4963a66550c95b7d7a55 2024-12-08T01:15:57,235 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/C of c0ffe97318107908ae81381c359cc9b5 into 79d6947af0ca4963a66550c95b7d7a55(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:15:57,235 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:57,235 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/C, priority=13, startTime=1733620556379; duration=0sec 2024-12-08T01:15:57,235 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:15:57,235 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:C 2024-12-08T01:15:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620617292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620617292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620617292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620617292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,351 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046 2024-12-08T01:15:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/f5c05ad86ac0486891453e5de1f31163 is 50, key is test_row_0/B:col10/1733620554776/Put/seqid=0 2024-12-08T01:15:57,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742206_1382 (size=12151) 2024-12-08T01:15:57,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620617601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620617601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620617602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620617602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:57,759 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/f5c05ad86ac0486891453e5de1f31163 2024-12-08T01:15:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/4efe3370507e45b182998ac4939fc68e is 50, key is test_row_0/C:col10/1733620554776/Put/seqid=0 2024-12-08T01:15:57,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742207_1383 (size=12151) 2024-12-08T01:15:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620618112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620618112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620618112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620618112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:58,168 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/4efe3370507e45b182998ac4939fc68e 2024-12-08T01:15:58,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046 2024-12-08T01:15:58,174 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046, entries=150, sequenceid=166, filesize=30.4 K 2024-12-08T01:15:58,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/f5c05ad86ac0486891453e5de1f31163 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f5c05ad86ac0486891453e5de1f31163 2024-12-08T01:15:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f5c05ad86ac0486891453e5de1f31163, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T01:15:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/4efe3370507e45b182998ac4939fc68e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4efe3370507e45b182998ac4939fc68e 2024-12-08T01:15:58,183 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4efe3370507e45b182998ac4939fc68e, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T01:15:58,183 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c0ffe97318107908ae81381c359cc9b5 in 1655ms, sequenceid=166, compaction requested=false 2024-12-08T01:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-08T01:15:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-08T01:15:58,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-08T01:15:58,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4190 sec 2024-12-08T01:15:58,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4230 sec 2024-12-08T01:15:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:59,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T01:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:15:59,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120833efc041481e449d8948b8fb7853d444_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620559124/Put/seqid=0 2024-12-08T01:15:59,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742208_1384 (size=14794) 2024-12-08T01:15:59,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:15:59,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120833efc041481e449d8948b8fb7853d444_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120833efc041481e449d8948b8fb7853d444_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:15:59,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/0313aefda67e4710a04ee16b05968adc, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:15:59,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/0313aefda67e4710a04ee16b05968adc is 175, key is test_row_0/A:col10/1733620559124/Put/seqid=0 2024-12-08T01:15:59,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742209_1385 (size=39749) 2024-12-08T01:15:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620619166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620559164, max timestamp=1733620559176Num uniques 859; 2024-12-08T01:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620559164, max timestamp=1733620559176Num uniques 859; 2024-12-08T01:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620559164, max timestamp=1733620559176Num uniques 859; 2024-12-08T01:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:15:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620619208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620619215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620619216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620619282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620619321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620619321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620619332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620619492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/ff0cd59264a149c88b1eda6640346346, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/99f8da3058d34f22bf3e9c1abe38af2c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a] to archive 2024-12-08T01:15:59,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:59,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/11395b55fe7b45c6ae8e625be886dd97 2024-12-08T01:15:59,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6036c6f23f1f43d681d61d4d6138f87c 2024-12-08T01:15:59,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/df5d7a4e1d3f4d7e9573f2de89ca037d 2024-12-08T01:15:59,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/ff0cd59264a149c88b1eda6640346346 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/ff0cd59264a149c88b1eda6640346346 2024-12-08T01:15:59,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/f9d94484b1b64ac2b127ac1d8fc0fe11 2024-12-08T01:15:59,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/e30328b20d7b429a8ab0002c70697716 2024-12-08T01:15:59,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/c7f513c11a994feab7aa10c4a3d0c8cd 2024-12-08T01:15:59,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/99f8da3058d34f22bf3e9c1abe38af2c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/99f8da3058d34f22bf3e9c1abe38af2c 2024-12-08T01:15:59,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/7115cdc39dbb4673a94799b93eeb3eec 2024-12-08T01:15:59,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/471283bab20c432d812d8288973b055a 2024-12-08T01:15:59,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4dde575a406046bcba081c102f07b3fd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/716dfbbb495a49fcb43fc65adfd44007, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/379fd2c0bda1467aa779d6d925437fb2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7248a1d8ffc34828a303b6badac81921, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7e09c6691b7a451dbaeb706592f9f133, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/57d34d13a3954f6d97d3a2b952defb9a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/697d5cc6a9ae45a1b609251a3ab90275, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/0e85399350ed4fe59ed15bfba7e97717, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/74f4ef151d6c47a3b164468dbe30e14b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4a126f3a840147aa94a1aba58e29d4a1] to archive 2024-12-08T01:15:59,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:59,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4dde575a406046bcba081c102f07b3fd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4dde575a406046bcba081c102f07b3fd 2024-12-08T01:15:59,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/716dfbbb495a49fcb43fc65adfd44007 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/716dfbbb495a49fcb43fc65adfd44007 2024-12-08T01:15:59,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/379fd2c0bda1467aa779d6d925437fb2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/379fd2c0bda1467aa779d6d925437fb2 2024-12-08T01:15:59,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7248a1d8ffc34828a303b6badac81921 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7248a1d8ffc34828a303b6badac81921 2024-12-08T01:15:59,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7e09c6691b7a451dbaeb706592f9f133 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/7e09c6691b7a451dbaeb706592f9f133 2024-12-08T01:15:59,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/57d34d13a3954f6d97d3a2b952defb9a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/57d34d13a3954f6d97d3a2b952defb9a 2024-12-08T01:15:59,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/697d5cc6a9ae45a1b609251a3ab90275 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/697d5cc6a9ae45a1b609251a3ab90275 2024-12-08T01:15:59,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/0e85399350ed4fe59ed15bfba7e97717 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/0e85399350ed4fe59ed15bfba7e97717 2024-12-08T01:15:59,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/74f4ef151d6c47a3b164468dbe30e14b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/74f4ef151d6c47a3b164468dbe30e14b 2024-12-08T01:15:59,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4a126f3a840147aa94a1aba58e29d4a1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/4a126f3a840147aa94a1aba58e29d4a1 2024-12-08T01:15:59,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/dbe1763ac4a14bb495e33fdcf5a92b55, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/31a7561deecd4d2c9038b6992e50ca6b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fcd4af1afc33402d8841e5ad86ca2547, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/c22a0f23078645568f7c3f6aa77e5aaf, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fc99d67e80dd49148f9ae8c0ccbc2cc8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/36ba888bfc254308b4a4f15d516f58fd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4d38ac6855634c31a7f2d93fbd6d1977, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7ee12f0048dc47edaa3683aae2e8dfc0] to archive 2024-12-08T01:15:59,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:15:59,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/dbe1763ac4a14bb495e33fdcf5a92b55 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/dbe1763ac4a14bb495e33fdcf5a92b55 2024-12-08T01:15:59,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/31a7561deecd4d2c9038b6992e50ca6b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/31a7561deecd4d2c9038b6992e50ca6b 2024-12-08T01:15:59,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fcd4af1afc33402d8841e5ad86ca2547 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fcd4af1afc33402d8841e5ad86ca2547 2024-12-08T01:15:59,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/c22a0f23078645568f7c3f6aa77e5aaf to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/c22a0f23078645568f7c3f6aa77e5aaf 2024-12-08T01:15:59,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fc99d67e80dd49148f9ae8c0ccbc2cc8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/fc99d67e80dd49148f9ae8c0ccbc2cc8 2024-12-08T01:15:59,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/2bbf42c0c2ea4bdc949a88a8cd79e2a0 2024-12-08T01:15:59,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/36ba888bfc254308b4a4f15d516f58fd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/36ba888bfc254308b4a4f15d516f58fd 2024-12-08T01:15:59,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4d38ac6855634c31a7f2d93fbd6d1977 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4d38ac6855634c31a7f2d93fbd6d1977 2024-12-08T01:15:59,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/8a7f2d8fc03d4e33a1500d33ab3fbaf5 2024-12-08T01:15:59,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7ee12f0048dc47edaa3683aae2e8dfc0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7ee12f0048dc47edaa3683aae2e8dfc0 2024-12-08T01:15:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620619531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620619531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620619542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/0313aefda67e4710a04ee16b05968adc 2024-12-08T01:15:59,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/04aef1af296349b5beda689d59590d72 is 50, key is test_row_0/B:col10/1733620559124/Put/seqid=0 2024-12-08T01:15:59,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742210_1386 (size=12151) 2024-12-08T01:15:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620619812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620619842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620619842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:15:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620619862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:15:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:15:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:15:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T01:15:59,923 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:15:59,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-08T01:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:15:59,924 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:15:59,925 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:15:59,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:15:59,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/04aef1af296349b5beda689d59590d72 2024-12-08T01:15:59,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/44c2729f066747fea9b5562277e7ce2a is 50, key is test_row_0/C:col10/1733620559124/Put/seqid=0 2024-12-08T01:15:59,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742211_1387 (size=12151) 2024-12-08T01:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:16:00,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T01:16:00,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T01:16:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:16:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620620332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:00,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620620352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620620352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:00,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T01:16:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:00,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620620382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:00,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/44c2729f066747fea9b5562277e7ce2a 2024-12-08T01:16:00,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/0313aefda67e4710a04ee16b05968adc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc 2024-12-08T01:16:00,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc, entries=200, sequenceid=182, filesize=38.8 K 2024-12-08T01:16:00,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/04aef1af296349b5beda689d59590d72 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/04aef1af296349b5beda689d59590d72 2024-12-08T01:16:00,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/04aef1af296349b5beda689d59590d72, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T01:16:00,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/44c2729f066747fea9b5562277e7ce2a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/44c2729f066747fea9b5562277e7ce2a 2024-12-08T01:16:00,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/44c2729f066747fea9b5562277e7ce2a, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T01:16:00,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c0ffe97318107908ae81381c359cc9b5 in 1284ms, sequenceid=182, compaction requested=true 2024-12-08T01:16:00,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:00,410 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:00,410 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:00,410 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:00,410 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:00,411 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/A is initiating minor compaction (all files) 2024-12-08T01:16:00,411 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/B is initiating minor compaction (all files) 2024-12-08T01:16:00,411 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/A in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,411 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/B in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,411 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/4e70e11984024164a2a568289687a396, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=99.9 K 2024-12-08T01:16:00,411 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/739b054ba6e94f22a5dec764f4dd73d6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f5c05ad86ac0486891453e5de1f31163, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/04aef1af296349b5beda689d59590d72] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.9 K 2024-12-08T01:16:00,411 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,411 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/4e70e11984024164a2a568289687a396, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc] 2024-12-08T01:16:00,412 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739b054ba6e94f22a5dec764f4dd73d6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733620553552 2024-12-08T01:16:00,412 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e70e11984024164a2a568289687a396, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733620553552 2024-12-08T01:16:00,412 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c05ad86ac0486891453e5de1f31163, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733620554768 2024-12-08T01:16:00,412 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9d39fdbf0b4edfb0e0aef1a6c0c046, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733620554768 2024-12-08T01:16:00,412 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04aef1af296349b5beda689d59590d72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733620556954 2024-12-08T01:16:00,412 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0313aefda67e4710a04ee16b05968adc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733620556954 2024-12-08T01:16:00,417 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:00,419 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:00,419 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/f81234ebe4954ecd84f062c385452797 is 50, key is test_row_0/B:col10/1733620559124/Put/seqid=0 2024-12-08T01:16:00,419 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208890fb8c09d98437da8c1a332d018934a_c0ffe97318107908ae81381c359cc9b5 store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:00,422 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208890fb8c09d98437da8c1a332d018934a_c0ffe97318107908ae81381c359cc9b5, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:00,423 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208890fb8c09d98437da8c1a332d018934a_c0ffe97318107908ae81381c359cc9b5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:00,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742212_1388 (size=12254) 2024-12-08T01:16:00,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742213_1389 (size=4469) 2024-12-08T01:16:00,436 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#A#compaction#331 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:00,436 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/5541049224d6492788c28b40fd9238ba is 175, key is test_row_0/A:col10/1733620559124/Put/seqid=0 2024-12-08T01:16:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742214_1390 (size=31208) 2024-12-08T01:16:00,448 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/5541049224d6492788c28b40fd9238ba as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/5541049224d6492788c28b40fd9238ba 2024-12-08T01:16:00,454 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/A of c0ffe97318107908ae81381c359cc9b5 into 5541049224d6492788c28b40fd9238ba(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:00,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:00,454 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/A, priority=13, startTime=1733620560410; duration=0sec 2024-12-08T01:16:00,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:00,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:A 2024-12-08T01:16:00,454 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:00,455 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:00,455 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/C is initiating minor compaction (all files) 2024-12-08T01:16:00,455 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/C in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,455 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/79d6947af0ca4963a66550c95b7d7a55, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4efe3370507e45b182998ac4939fc68e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/44c2729f066747fea9b5562277e7ce2a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.9 K 2024-12-08T01:16:00,455 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 79d6947af0ca4963a66550c95b7d7a55, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733620553552 2024-12-08T01:16:00,455 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4efe3370507e45b182998ac4939fc68e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733620554768 2024-12-08T01:16:00,456 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c2729f066747fea9b5562277e7ce2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733620556954 2024-12-08T01:16:00,461 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:00,461 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/27d8e5595f5543b98743fae8c3722d0e is 50, key is test_row_0/C:col10/1733620559124/Put/seqid=0 2024-12-08T01:16:00,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742215_1391 (size=12254) 2024-12-08T01:16:00,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:00,535 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d8e2dc5f7ef04f02ae5ee8a0ee6bff3b_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620559214/Put/seqid=0 2024-12-08T01:16:00,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742216_1392 (size=12304) 2024-12-08T01:16:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:16:00,834 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/f81234ebe4954ecd84f062c385452797 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f81234ebe4954ecd84f062c385452797 2024-12-08T01:16:00,837 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/B of c0ffe97318107908ae81381c359cc9b5 into f81234ebe4954ecd84f062c385452797(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:00,837 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:00,837 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/B, priority=13, startTime=1733620560410; duration=0sec 2024-12-08T01:16:00,838 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:00,838 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:B 2024-12-08T01:16:00,868 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/27d8e5595f5543b98743fae8c3722d0e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/27d8e5595f5543b98743fae8c3722d0e 2024-12-08T01:16:00,872 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/C of c0ffe97318107908ae81381c359cc9b5 into 27d8e5595f5543b98743fae8c3722d0e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:00,872 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:00,872 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/C, priority=13, startTime=1733620560410; duration=0sec 2024-12-08T01:16:00,872 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:00,872 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:C 2024-12-08T01:16:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:00,949 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d8e2dc5f7ef04f02ae5ee8a0ee6bff3b_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d8e2dc5f7ef04f02ae5ee8a0ee6bff3b_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1dfa8d5e3a684f349a311551c898ac5f, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:00,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1dfa8d5e3a684f349a311551c898ac5f is 175, key is test_row_0/A:col10/1733620559214/Put/seqid=0 2024-12-08T01:16:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742217_1393 (size=31105) 2024-12-08T01:16:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:16:01,353 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1dfa8d5e3a684f349a311551c898ac5f 2024-12-08T01:16:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:01,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/a4cde2a141784216b07cb6449bfb5b4f is 50, key is test_row_0/B:col10/1733620559214/Put/seqid=0 2024-12-08T01:16:01,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742218_1394 (size=12151) 2024-12-08T01:16:01,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620621415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620621416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620621417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620621417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620621532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620621532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620621532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620621532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620621743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620621743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620621743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620621743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:01,764 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/a4cde2a141784216b07cb6449bfb5b4f 2024-12-08T01:16:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/a5a1385acb9143079d160d50d977d5df is 50, key is test_row_0/C:col10/1733620559214/Put/seqid=0 2024-12-08T01:16:01,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742219_1395 (size=12151) 2024-12-08T01:16:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620622062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620622062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620622062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620622062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:16:02,176 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/a5a1385acb9143079d160d50d977d5df 2024-12-08T01:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/1dfa8d5e3a684f349a311551c898ac5f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f 2024-12-08T01:16:02,181 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f, entries=150, sequenceid=217, filesize=30.4 K 2024-12-08T01:16:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/a4cde2a141784216b07cb6449bfb5b4f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/a4cde2a141784216b07cb6449bfb5b4f 2024-12-08T01:16:02,184 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/a4cde2a141784216b07cb6449bfb5b4f, entries=150, sequenceid=217, filesize=11.9 K 2024-12-08T01:16:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/a5a1385acb9143079d160d50d977d5df as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/a5a1385acb9143079d160d50d977d5df 2024-12-08T01:16:02,191 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/a5a1385acb9143079d160d50d977d5df, entries=150, sequenceid=217, filesize=11.9 K 2024-12-08T01:16:02,191 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=73.80 KB/75570 for c0ffe97318107908ae81381c359cc9b5 in 1656ms, sequenceid=217, compaction requested=false 2024-12-08T01:16:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-08T01:16:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-08T01:16:02,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-08T01:16:02,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2680 sec 2024-12-08T01:16:02,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2720 sec 2024-12-08T01:16:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:02,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:16:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:16:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:16:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:16:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:02,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b17eb4db016b4211a5491898e2aff897_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:02,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742220_1396 (size=14794) 2024-12-08T01:16:02,505 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:02,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b17eb4db016b4211a5491898e2aff897_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b17eb4db016b4211a5491898e2aff897_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:02,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/49eab8ad8ebf4266a6e0ea33dd831325, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:02,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/49eab8ad8ebf4266a6e0ea33dd831325 is 175, key is test_row_0/A:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:02,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742221_1397 (size=39749) 2024-12-08T01:16:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620622582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620622582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620622583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620622584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620622585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620622701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620622701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620622702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/49eab8ad8ebf4266a6e0ea33dd831325 2024-12-08T01:16:02,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/d22cb79e8d4348a1b6527eca042e6ac7 is 50, key is test_row_0/B:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620622912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620622912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620622912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:02,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742222_1398 (size=12151) 2024-12-08T01:16:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620623222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620623222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620623232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/d22cb79e8d4348a1b6527eca042e6ac7 2024-12-08T01:16:03,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/eb7b04d709244b15a59869a7a5a3b0ab is 50, key is test_row_0/C:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:03,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742223_1399 (size=12151) 2024-12-08T01:16:03,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44412 deadline: 1733620623592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,597 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in c0ffe97318107908ae81381c359cc9b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44336 deadline: 1733620623602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,607 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in c0ffe97318107908ae81381c359cc9b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/eb7b04d709244b15a59869a7a5a3b0ab 2024-12-08T01:16:03,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/49eab8ad8ebf4266a6e0ea33dd831325 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325 2024-12-08T01:16:03,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325, entries=200, sequenceid=234, filesize=38.8 K 2024-12-08T01:16:03,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/d22cb79e8d4348a1b6527eca042e6ac7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/d22cb79e8d4348a1b6527eca042e6ac7 2024-12-08T01:16:03,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620623732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/d22cb79e8d4348a1b6527eca042e6ac7, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T01:16:03,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/eb7b04d709244b15a59869a7a5a3b0ab as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/eb7b04d709244b15a59869a7a5a3b0ab 2024-12-08T01:16:03,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/eb7b04d709244b15a59869a7a5a3b0ab, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T01:16:03,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0ffe97318107908ae81381c359cc9b5 in 1248ms, sequenceid=234, compaction requested=true 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:03,744 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:03,744 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0ffe97318107908ae81381c359cc9b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/B is initiating minor compaction (all files) 2024-12-08T01:16:03,745 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/B in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:03,745 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f81234ebe4954ecd84f062c385452797, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/a4cde2a141784216b07cb6449bfb5b4f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/d22cb79e8d4348a1b6527eca042e6ac7] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.7 K 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102062 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/A is initiating minor compaction (all files) 2024-12-08T01:16:03,745 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/A in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:03,745 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/5541049224d6492788c28b40fd9238ba, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=99.7 K 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f81234ebe4954ecd84f062c385452797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733620556954 2024-12-08T01:16:03,745 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/5541049224d6492788c28b40fd9238ba, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325] 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a4cde2a141784216b07cb6449bfb5b4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733620559207 2024-12-08T01:16:03,745 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5541049224d6492788c28b40fd9238ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733620556954 2024-12-08T01:16:03,746 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d22cb79e8d4348a1b6527eca042e6ac7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733620561377 2024-12-08T01:16:03,746 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dfa8d5e3a684f349a311551c898ac5f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733620559207 2024-12-08T01:16:03,746 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49eab8ad8ebf4266a6e0ea33dd831325, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733620561377 2024-12-08T01:16:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:03,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:16:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:16:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:16:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:16:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:03,753 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:03,755 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:03,755 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412086cdbbf8ec46d4ef9bdf62c227ad74ec6_c0ffe97318107908ae81381c359cc9b5 store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:03,755 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/1acfed7fa9924126af2779dd849a6865 is 50, key is test_row_0/B:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:03,757 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412086cdbbf8ec46d4ef9bdf62c227ad74ec6_c0ffe97318107908ae81381c359cc9b5, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:03,757 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086cdbbf8ec46d4ef9bdf62c227ad74ec6_c0ffe97318107908ae81381c359cc9b5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:03,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742224_1400 (size=12357) 2024-12-08T01:16:03,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208cdb09f3c01ea4b12ba1e3ac786e95b10_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620562583/Put/seqid=0 2024-12-08T01:16:03,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742225_1401 (size=4469) 2024-12-08T01:16:03,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742226_1402 (size=14794) 2024-12-08T01:16:03,780 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/1acfed7fa9924126af2779dd849a6865 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/1acfed7fa9924126af2779dd849a6865 2024-12-08T01:16:03,786 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/B of c0ffe97318107908ae81381c359cc9b5 into 1acfed7fa9924126af2779dd849a6865(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:03,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:03,787 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/B, priority=13, startTime=1733620563744; duration=0sec 2024-12-08T01:16:03,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:03,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:B 2024-12-08T01:16:03,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:03,787 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:03,788 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): c0ffe97318107908ae81381c359cc9b5/C is initiating minor compaction (all files) 2024-12-08T01:16:03,788 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0ffe97318107908ae81381c359cc9b5/C in TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:03,788 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/27d8e5595f5543b98743fae8c3722d0e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/a5a1385acb9143079d160d50d977d5df, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/eb7b04d709244b15a59869a7a5a3b0ab] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp, totalSize=35.7 K 2024-12-08T01:16:03,788 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d8e5595f5543b98743fae8c3722d0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733620556954 2024-12-08T01:16:03,788 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a1385acb9143079d160d50d977d5df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733620559207 2024-12-08T01:16:03,788 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7b04d709244b15a59869a7a5a3b0ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733620561377 2024-12-08T01:16:03,798 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:03,798 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/ec360031d66040eca016f8b8d9547dd2 is 50, key is test_row_0/C:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:03,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742227_1403 (size=12357) 2024-12-08T01:16:03,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620623796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620623806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620623912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:03,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620623922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T01:16:04,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:04,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-08T01:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:04,085 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:04,086 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:04,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620624133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620624142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,166 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0ffe97318107908ae81381c359cc9b5#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:04,167 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/6c22e41055284c4c910e7628ea7078a9 is 175, key is test_row_0/A:col10/1733620561414/Put/seqid=0 2024-12-08T01:16:04,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742228_1404 (size=31311) 2024-12-08T01:16:04,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:04,180 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208cdb09f3c01ea4b12ba1e3ac786e95b10_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208cdb09f3c01ea4b12ba1e3ac786e95b10_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:04,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/3c2cc5100abb4e57a6f9b60e506b7a6e, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:04,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/3c2cc5100abb4e57a6f9b60e506b7a6e is 175, key is test_row_0/A:col10/1733620562583/Put/seqid=0 2024-12-08T01:16:04,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742229_1405 (size=39749) 2024-12-08T01:16:04,185 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-12-08T01:16:04,185 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:04,185 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,185 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:04,187 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-12-08T01:16:04,187 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:04,187 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,187 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:04,187 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-12-08T01:16:04,187 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:04,187 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,187 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:04,194 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-12-08T01:16:04,194 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:04,194 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,194 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:04,195 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-12-08T01:16:04,195 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:04,195 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,195 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:04,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:04,206 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/ec360031d66040eca016f8b8d9547dd2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/ec360031d66040eca016f8b8d9547dd2 2024-12-08T01:16:04,210 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/C of c0ffe97318107908ae81381c359cc9b5 into ec360031d66040eca016f8b8d9547dd2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:04,210 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:04,210 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/C, priority=13, startTime=1733620563744; duration=0sec 2024-12-08T01:16:04,210 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:04,210 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:C 2024-12-08T01:16:04,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:04,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620624442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620624452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,575 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/6c22e41055284c4c910e7628ea7078a9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6c22e41055284c4c910e7628ea7078a9 2024-12-08T01:16:04,578 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0ffe97318107908ae81381c359cc9b5/A of c0ffe97318107908ae81381c359cc9b5 into 6c22e41055284c4c910e7628ea7078a9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:04,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:04,578 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., storeName=c0ffe97318107908ae81381c359cc9b5/A, priority=13, startTime=1733620563744; duration=0sec 2024-12-08T01:16:04,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:04,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0ffe97318107908ae81381c359cc9b5:A 2024-12-08T01:16:04,584 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/3c2cc5100abb4e57a6f9b60e506b7a6e 2024-12-08T01:16:04,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/65ce6242d6b24a588d3b640cbf4e812d is 50, key is test_row_0/B:col10/1733620562583/Put/seqid=0 2024-12-08T01:16:04,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742230_1406 (size=12151) 2024-12-08T01:16:04,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44408 deadline: 1733620624743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,745 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in c0ffe97318107908ae81381c359cc9b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:04,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:04,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44382 deadline: 1733620624952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44376 deadline: 1733620624963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 2024-12-08T01:16:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:04,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/65ce6242d6b24a588d3b640cbf4e812d 2024-12-08T01:16:05,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/68cba9499b754c2489206b522f8bc595 is 50, key is test_row_0/C:col10/1733620562583/Put/seqid=0 2024-12-08T01:16:05,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742231_1407 (size=12151) 2024-12-08T01:16:05,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:05,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:05,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:05,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:05,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:05,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:05,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/68cba9499b754c2489206b522f8bc595 2024-12-08T01:16:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/3c2cc5100abb4e57a6f9b60e506b7a6e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/3c2cc5100abb4e57a6f9b60e506b7a6e 2024-12-08T01:16:05,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/3c2cc5100abb4e57a6f9b60e506b7a6e, entries=200, sequenceid=252, filesize=38.8 K 2024-12-08T01:16:05,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/65ce6242d6b24a588d3b640cbf4e812d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/65ce6242d6b24a588d3b640cbf4e812d 2024-12-08T01:16:05,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/65ce6242d6b24a588d3b640cbf4e812d, entries=150, sequenceid=252, filesize=11.9 K 2024-12-08T01:16:05,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/68cba9499b754c2489206b522f8bc595 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/68cba9499b754c2489206b522f8bc595 2024-12-08T01:16:05,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/68cba9499b754c2489206b522f8bc595, entries=150, sequenceid=252, filesize=11.9 K 2024-12-08T01:16:05,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0ffe97318107908ae81381c359cc9b5 in 1676ms, sequenceid=252, compaction requested=false 2024-12-08T01:16:05,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:05,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:05,484 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083df9293c7cb74f9c8ec8ba31e0d321b4_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620563794/Put/seqid=0 2024-12-08T01:16:05,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742232_1408 (size=12454) 2024-12-08T01:16:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:05,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. as already flushing 2024-12-08T01:16:05,615 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-12-08T01:16:05,615 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:05,615 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,615 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:05,624 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-12-08T01:16:05,624 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:05,624 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,625 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:05,899 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083df9293c7cb74f9c8ec8ba31e0d321b4_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083df9293c7cb74f9c8ec8ba31e0d321b4_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/8db7a772ecd445a4a826d6b2bc99b98e, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/8db7a772ecd445a4a826d6b2bc99b98e is 175, key is test_row_0/A:col10/1733620563794/Put/seqid=0 2024-12-08T01:16:05,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742233_1409 (size=31255) 2024-12-08T01:16:05,966 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-12-08T01:16:05,966 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:05,967 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,967 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:05,984 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-12-08T01:16:05,984 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:05,984 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,985 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:05,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:06,308 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/8db7a772ecd445a4a826d6b2bc99b98e 2024-12-08T01:16:06,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/df9ad39431564e9d8d52cd026f959374 is 50, key is test_row_0/B:col10/1733620563794/Put/seqid=0 2024-12-08T01:16:06,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742234_1410 (size=12301) 2024-12-08T01:16:06,322 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/df9ad39431564e9d8d52cd026f959374 2024-12-08T01:16:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/7f8e67abcd594a1896fa4c95a49959a5 is 50, key is test_row_0/C:col10/1733620563794/Put/seqid=0 2024-12-08T01:16:06,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742235_1411 (size=12301) 2024-12-08T01:16:06,737 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/7f8e67abcd594a1896fa4c95a49959a5 2024-12-08T01:16:06,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/8db7a772ecd445a4a826d6b2bc99b98e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/8db7a772ecd445a4a826d6b2bc99b98e 2024-12-08T01:16:06,745 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/8db7a772ecd445a4a826d6b2bc99b98e, entries=150, sequenceid=269, filesize=30.5 K 2024-12-08T01:16:06,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/df9ad39431564e9d8d52cd026f959374 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/df9ad39431564e9d8d52cd026f959374 2024-12-08T01:16:06,749 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/df9ad39431564e9d8d52cd026f959374, entries=150, sequenceid=269, filesize=12.0 K 2024-12-08T01:16:06,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/7f8e67abcd594a1896fa4c95a49959a5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7f8e67abcd594a1896fa4c95a49959a5 2024-12-08T01:16:06,753 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7f8e67abcd594a1896fa4c95a49959a5, entries=150, sequenceid=269, filesize=12.0 K 2024-12-08T01:16:06,754 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for c0ffe97318107908ae81381c359cc9b5 in 1270ms, sequenceid=269, compaction requested=true 2024-12-08T01:16:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for c0ffe97318107908ae81381c359cc9b5: 2024-12-08T01:16:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-08T01:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-08T01:16:06,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-08T01:16:06,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6690 sec 2024-12-08T01:16:06,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6750 sec 2024-12-08T01:16:06,764 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-12-08T01:16:06,764 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:06,764 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:06,764 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:06,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T01:16:08,244 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:08,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T01:16:08,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-08T01:16:08,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-08T01:16:08,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-08T01:16:08,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-08T01:16:08,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1655 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4965 rows 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1653 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4956 rows 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1654 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4962 rows 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1657 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4971 rows 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1662 2024-12-08T01:16:08,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4983 rows 2024-12-08T01:16:08,245 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:16:08,246 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:16:08,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:08,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:08,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:08,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T01:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T01:16:08,254 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620568254"}]},"ts":"1733620568254"} 2024-12-08T01:16:08,255 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T01:16:08,255 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T01:16:08,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:16:08,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, UNASSIGN}] 2024-12-08T01:16:08,259 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, UNASSIGN 2024-12-08T01:16:08,259 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:08,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, UNASSIGN because future has completed 2024-12-08T01:16:08,261 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:16:08,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630}] 2024-12-08T01:16:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T01:16:08,413 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:08,413 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:16:08,413 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing c0ffe97318107908ae81381c359cc9b5, disabling compactions & flushes 2024-12-08T01:16:08,413 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. after waiting 0 ms 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:08,414 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing c0ffe97318107908ae81381c359cc9b5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=A 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=B 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0ffe97318107908ae81381c359cc9b5, store=C 2024-12-08T01:16:08,414 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:08,422 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088bd32bdcc2124058aa66bf4fd9705843_c0ffe97318107908ae81381c359cc9b5 is 50, key is test_row_0/A:col10/1733620565982/Put/seqid=0 2024-12-08T01:16:08,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742236_1412 (size=12454) 2024-12-08T01:16:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T01:16:08,825 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:08,829 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088bd32bdcc2124058aa66bf4fd9705843_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088bd32bdcc2124058aa66bf4fd9705843_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:08,830 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/dab541d074d2486aa336e9dde2dc97a3, store: [table=TestAcidGuarantees family=A region=c0ffe97318107908ae81381c359cc9b5] 2024-12-08T01:16:08,830 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/dab541d074d2486aa336e9dde2dc97a3 is 175, key is test_row_0/A:col10/1733620565982/Put/seqid=0 2024-12-08T01:16:08,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742237_1413 (size=31255) 2024-12-08T01:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T01:16:09,234 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/dab541d074d2486aa336e9dde2dc97a3 2024-12-08T01:16:09,245 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/38d2139650b54b4b955dc275482ae87e is 50, key is test_row_0/B:col10/1733620565982/Put/seqid=0 2024-12-08T01:16:09,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742238_1414 (size=12301) 2024-12-08T01:16:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T01:16:09,649 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/38d2139650b54b4b955dc275482ae87e 2024-12-08T01:16:09,655 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/1284e47e13d84edba349fd4253ee6303 is 50, key is test_row_0/C:col10/1733620565982/Put/seqid=0 2024-12-08T01:16:09,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742239_1415 (size=12301) 2024-12-08T01:16:10,059 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/1284e47e13d84edba349fd4253ee6303 2024-12-08T01:16:10,063 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/A/dab541d074d2486aa336e9dde2dc97a3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/dab541d074d2486aa336e9dde2dc97a3 2024-12-08T01:16:10,066 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/dab541d074d2486aa336e9dde2dc97a3, entries=150, sequenceid=277, filesize=30.5 K 2024-12-08T01:16:10,067 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/B/38d2139650b54b4b955dc275482ae87e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/38d2139650b54b4b955dc275482ae87e 2024-12-08T01:16:10,070 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/38d2139650b54b4b955dc275482ae87e, entries=150, sequenceid=277, filesize=12.0 K 2024-12-08T01:16:10,071 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/.tmp/C/1284e47e13d84edba349fd4253ee6303 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/1284e47e13d84edba349fd4253ee6303 2024-12-08T01:16:10,074 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/1284e47e13d84edba349fd4253ee6303, entries=150, sequenceid=277, filesize=12.0 K 2024-12-08T01:16:10,075 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0ffe97318107908ae81381c359cc9b5 in 1660ms, sequenceid=277, compaction requested=true 2024-12-08T01:16:10,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/4e70e11984024164a2a568289687a396, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/5541049224d6492788c28b40fd9238ba, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325] to archive 2024-12-08T01:16:10,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:16:10,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/4e70e11984024164a2a568289687a396 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/4e70e11984024164a2a568289687a396 2024-12-08T01:16:10,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1a9d39fdbf0b4edfb0e0aef1a6c0c046 2024-12-08T01:16:10,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/0313aefda67e4710a04ee16b05968adc 2024-12-08T01:16:10,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/5541049224d6492788c28b40fd9238ba to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/5541049224d6492788c28b40fd9238ba 2024-12-08T01:16:10,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/1dfa8d5e3a684f349a311551c898ac5f 2024-12-08T01:16:10,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/49eab8ad8ebf4266a6e0ea33dd831325 2024-12-08T01:16:10,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/739b054ba6e94f22a5dec764f4dd73d6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f5c05ad86ac0486891453e5de1f31163, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f81234ebe4954ecd84f062c385452797, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/04aef1af296349b5beda689d59590d72, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/a4cde2a141784216b07cb6449bfb5b4f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/d22cb79e8d4348a1b6527eca042e6ac7] to archive 2024-12-08T01:16:10,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:16:10,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/739b054ba6e94f22a5dec764f4dd73d6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/739b054ba6e94f22a5dec764f4dd73d6 2024-12-08T01:16:10,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f5c05ad86ac0486891453e5de1f31163 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f5c05ad86ac0486891453e5de1f31163 2024-12-08T01:16:10,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f81234ebe4954ecd84f062c385452797 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/f81234ebe4954ecd84f062c385452797 2024-12-08T01:16:10,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/04aef1af296349b5beda689d59590d72 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/04aef1af296349b5beda689d59590d72 2024-12-08T01:16:10,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/a4cde2a141784216b07cb6449bfb5b4f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/a4cde2a141784216b07cb6449bfb5b4f 2024-12-08T01:16:10,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/d22cb79e8d4348a1b6527eca042e6ac7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/d22cb79e8d4348a1b6527eca042e6ac7 2024-12-08T01:16:10,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/79d6947af0ca4963a66550c95b7d7a55, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4efe3370507e45b182998ac4939fc68e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/27d8e5595f5543b98743fae8c3722d0e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/44c2729f066747fea9b5562277e7ce2a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/a5a1385acb9143079d160d50d977d5df, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/eb7b04d709244b15a59869a7a5a3b0ab] to archive 2024-12-08T01:16:10,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:16:10,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/79d6947af0ca4963a66550c95b7d7a55 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/79d6947af0ca4963a66550c95b7d7a55 2024-12-08T01:16:10,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4efe3370507e45b182998ac4939fc68e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/4efe3370507e45b182998ac4939fc68e 2024-12-08T01:16:10,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/27d8e5595f5543b98743fae8c3722d0e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/27d8e5595f5543b98743fae8c3722d0e 2024-12-08T01:16:10,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/44c2729f066747fea9b5562277e7ce2a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/44c2729f066747fea9b5562277e7ce2a 2024-12-08T01:16:10,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/a5a1385acb9143079d160d50d977d5df to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/a5a1385acb9143079d160d50d977d5df 2024-12-08T01:16:10,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/eb7b04d709244b15a59869a7a5a3b0ab to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/eb7b04d709244b15a59869a7a5a3b0ab 2024-12-08T01:16:10,098 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/recovered.edits/280.seqid, newMaxSeqId=280, maxSeqId=4 2024-12-08T01:16:10,099 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. 2024-12-08T01:16:10,099 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for c0ffe97318107908ae81381c359cc9b5: Waiting for close lock at 1733620568413Running coprocessor pre-close hooks at 1733620568413Disabling compacts and flushes for region at 1733620568413Disabling writes for close at 1733620568414 (+1 ms)Obtaining lock to block concurrent updates at 1733620568414Preparing flush snapshotting stores in c0ffe97318107908ae81381c359cc9b5 at 1733620568414Finished memstore snapshotting TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733620568415 (+1 ms)Flushing stores of TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5. at 1733620568415Flushing c0ffe97318107908ae81381c359cc9b5/A: creating writer at 1733620568415Flushing mob file c0ffe97318107908ae81381c359cc9b5/A: appending metadata at 1733620568421 (+6 ms)Flushing mob file c0ffe97318107908ae81381c359cc9b5/A: closing flushed file at 1733620568422 (+1 ms)Flushing c0ffe97318107908ae81381c359cc9b5/A: appending metadata at 1733620568830 (+408 ms)Flushing c0ffe97318107908ae81381c359cc9b5/A: closing flushed file at 1733620568830Flushing c0ffe97318107908ae81381c359cc9b5/B: creating writer at 1733620569241 (+411 ms)Flushing c0ffe97318107908ae81381c359cc9b5/B: appending metadata at 1733620569245 (+4 ms)Flushing c0ffe97318107908ae81381c359cc9b5/B: closing flushed file at 1733620569245Flushing c0ffe97318107908ae81381c359cc9b5/C: creating writer at 1733620569652 (+407 ms)Flushing c0ffe97318107908ae81381c359cc9b5/C: appending metadata at 1733620569655 (+3 ms)Flushing c0ffe97318107908ae81381c359cc9b5/C: closing flushed file at 1733620569655Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23aa9c47: reopening flushed file at 1733620570062 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4db6bdfa: reopening flushed file at 1733620570066 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475fa22a: reopening flushed file at 1733620570070 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0ffe97318107908ae81381c359cc9b5 in 1660ms, sequenceid=277, compaction requested=true at 1733620570075 (+5 ms)Writing region close event to WAL at 1733620570096 (+21 ms)Running coprocessor post-close hooks at 1733620570099 (+3 ms)Closed at 1733620570099 2024-12-08T01:16:10,100 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,101 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c0ffe97318107908ae81381c359cc9b5, regionState=CLOSED 2024-12-08T01:16:10,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:16:10,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-08T01:16:10,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure c0ffe97318107908ae81381c359cc9b5, server=96982154c400,44843,1733620436630 in 1.8420 sec 2024-12-08T01:16:10,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-08T01:16:10,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0ffe97318107908ae81381c359cc9b5, UNASSIGN in 1.8460 sec 2024-12-08T01:16:10,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-08T01:16:10,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8500 sec 2024-12-08T01:16:10,108 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620570108"}]},"ts":"1733620570108"} 2024-12-08T01:16:10,109 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T01:16:10,109 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T01:16:10,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-12-08T01:16:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T01:16:10,412 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:10,413 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T01:16:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,416 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T01:16:10,418 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,422 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,424 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/recovered.edits] 2024-12-08T01:16:10,427 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/3c2cc5100abb4e57a6f9b60e506b7a6e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/3c2cc5100abb4e57a6f9b60e506b7a6e 2024-12-08T01:16:10,428 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6c22e41055284c4c910e7628ea7078a9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/6c22e41055284c4c910e7628ea7078a9 2024-12-08T01:16:10,429 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/8db7a772ecd445a4a826d6b2bc99b98e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/8db7a772ecd445a4a826d6b2bc99b98e 2024-12-08T01:16:10,430 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/dab541d074d2486aa336e9dde2dc97a3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/A/dab541d074d2486aa336e9dde2dc97a3 2024-12-08T01:16:10,432 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/1acfed7fa9924126af2779dd849a6865 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/1acfed7fa9924126af2779dd849a6865 2024-12-08T01:16:10,433 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/38d2139650b54b4b955dc275482ae87e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/38d2139650b54b4b955dc275482ae87e 2024-12-08T01:16:10,435 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/65ce6242d6b24a588d3b640cbf4e812d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/65ce6242d6b24a588d3b640cbf4e812d 2024-12-08T01:16:10,436 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/df9ad39431564e9d8d52cd026f959374 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/B/df9ad39431564e9d8d52cd026f959374 2024-12-08T01:16:10,439 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/1284e47e13d84edba349fd4253ee6303 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/1284e47e13d84edba349fd4253ee6303 2024-12-08T01:16:10,440 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/68cba9499b754c2489206b522f8bc595 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/68cba9499b754c2489206b522f8bc595 2024-12-08T01:16:10,441 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7f8e67abcd594a1896fa4c95a49959a5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/7f8e67abcd594a1896fa4c95a49959a5 2024-12-08T01:16:10,442 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/ec360031d66040eca016f8b8d9547dd2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/C/ec360031d66040eca016f8b8d9547dd2 2024-12-08T01:16:10,445 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/recovered.edits/280.seqid to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5/recovered.edits/280.seqid 2024-12-08T01:16:10,446 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,446 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T01:16:10,446 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T01:16:10,447 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T01:16:10,451 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120817be125e942840c6ae826331ae56d1cd_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120817be125e942840c6ae826331ae56d1cd_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,453 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083233a40390ec4accb160b34f48971c3c_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083233a40390ec4accb160b34f48971c3c_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,454 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120833efc041481e449d8948b8fb7853d444_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120833efc041481e449d8948b8fb7853d444_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,456 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083df9293c7cb74f9c8ec8ba31e0d321b4_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083df9293c7cb74f9c8ec8ba31e0d321b4_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,457 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120859366b9493174bad95a0052b2533efee_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120859366b9493174bad95a0052b2533efee_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,458 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085abf9b5d4f994c47af956846a7385f1c_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085abf9b5d4f994c47af956846a7385f1c_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,458 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087b3931695d7d4912831838d1e7a4c461_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087b3931695d7d4912831838d1e7a4c461_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,459 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088bd32bdcc2124058aa66bf4fd9705843_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088bd32bdcc2124058aa66bf4fd9705843_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,460 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b13abc1ff05f4e27822907f973772038_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b13abc1ff05f4e27822907f973772038_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,460 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b17eb4db016b4211a5491898e2aff897_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b17eb4db016b4211a5491898e2aff897_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,461 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208cdb09f3c01ea4b12ba1e3ac786e95b10_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208cdb09f3c01ea4b12ba1e3ac786e95b10_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,462 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d8e2dc5f7ef04f02ae5ee8a0ee6bff3b_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d8e2dc5f7ef04f02ae5ee8a0ee6bff3b_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,462 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ea9dc5f2d38e4a5bb044e30339482d0c_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ea9dc5f2d38e4a5bb044e30339482d0c_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,463 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f724fb1f994a4421a125e881f7693a1e_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f724fb1f994a4421a125e881f7693a1e_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,464 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f7b245bea8cd4d0bbb685575165e9a51_c0ffe97318107908ae81381c359cc9b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f7b245bea8cd4d0bbb685575165e9a51_c0ffe97318107908ae81381c359cc9b5 2024-12-08T01:16:10,464 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T01:16:10,465 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,467 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T01:16:10,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T01:16:10,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T01:16:10,470 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733620570469"}]},"ts":"9223372036854775807"} 2024-12-08T01:16:10,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T01:16:10,471 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c0ffe97318107908ae81381c359cc9b5, NAME => 'TestAcidGuarantees,,1733620540957.c0ffe97318107908ae81381c359cc9b5.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T01:16:10,471 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T01:16:10,471 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733620570471"}]},"ts":"9223372036854775807"} 2024-12-08T01:16:10,473 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T01:16:10,473 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-08T01:16:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T01:16:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T01:16:10,522 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:10,530 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-207628044_22 at /127.0.0.1:51302 [Waiting for operation #388] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_411094594_22 at /127.0.0.1:57218 [Waiting for operation #272] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_411094594_22 at /127.0.0.1:57220 [Waiting for operation #260] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-207628044_22 at /127.0.0.1:57214 [Waiting for operation #272] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 408) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10408 (was 9908) - AvailableMemoryMB LEAK? - 2024-12-08T01:16:10,539 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=10408 2024-12-08T01:16:10,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:16:10,541 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:16:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:10,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:16:10,543 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:10,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-12-08T01:16:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T01:16:10,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:16:10,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742240_1416 (size=1061) 2024-12-08T01:16:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T01:16:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T01:16:10,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:16:10,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742241_1417 (size=53) 2024-12-08T01:16:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T01:16:11,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:16:11,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 192cb468141f6e111e56d290a9940203, disabling compactions & flushes 2024-12-08T01:16:11,358 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. after waiting 0 ms 2024-12-08T01:16:11,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,358 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 192cb468141f6e111e56d290a9940203: Waiting for close lock at 1733620571358Disabling compacts and flushes for region at 1733620571358Disabling writes for close at 1733620571358Writing region close event to WAL at 1733620571358Closed at 1733620571358 2024-12-08T01:16:11,361 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:16:11,361 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733620571361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620571361"}]},"ts":"1733620571361"} 2024-12-08T01:16:11,364 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:16:11,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:16:11,366 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620571366"}]},"ts":"1733620571366"} 2024-12-08T01:16:11,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T01:16:11,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, ASSIGN}] 2024-12-08T01:16:11,371 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, ASSIGN 2024-12-08T01:16:11,372 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, ASSIGN; state=OFFLINE, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=false 2024-12-08T01:16:11,522 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=192cb468141f6e111e56d290a9940203, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:11,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, ASSIGN because future has completed 2024-12-08T01:16:11,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630}] 2024-12-08T01:16:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T01:16:11,684 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,685 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:16:11,686 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,686 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:16:11,686 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,686 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,688 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,689 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:11,689 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192cb468141f6e111e56d290a9940203 columnFamilyName A 2024-12-08T01:16:11,689 DEBUG [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:11,689 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.HStore(327): Store=192cb468141f6e111e56d290a9940203/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:11,689 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,691 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:11,691 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192cb468141f6e111e56d290a9940203 columnFamilyName B 2024-12-08T01:16:11,691 DEBUG [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:11,691 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.HStore(327): Store=192cb468141f6e111e56d290a9940203/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:11,691 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,692 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:11,693 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192cb468141f6e111e56d290a9940203 columnFamilyName C 2024-12-08T01:16:11,693 DEBUG [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:11,693 INFO [StoreOpener-192cb468141f6e111e56d290a9940203-1 {}] regionserver.HStore(327): Store=192cb468141f6e111e56d290a9940203/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:11,693 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,693 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,694 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,694 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,695 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,695 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,695 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:16:11,696 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,697 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:16:11,698 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 192cb468141f6e111e56d290a9940203; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74469879, jitterRate=0.1096876710653305}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:16:11,698 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:11,698 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 192cb468141f6e111e56d290a9940203: Running coprocessor pre-open hook at 1733620571686Writing region info on filesystem at 1733620571686Initializing all the Stores at 1733620571687 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620571687Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620571687Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620571688 (+1 ms)Cleaning up temporary data from old regions at 1733620571695 (+7 ms)Running coprocessor post-open hooks at 1733620571698 (+3 ms)Region opened successfully at 1733620571698 2024-12-08T01:16:11,699 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., pid=118, masterSystemTime=1733620571678 2024-12-08T01:16:11,700 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,700 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:11,701 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=192cb468141f6e111e56d290a9940203, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:11,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:16:11,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-08T01:16:11,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 in 178 msec 2024-12-08T01:16:11,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-08T01:16:11,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, ASSIGN in 336 msec 2024-12-08T01:16:11,707 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:16:11,707 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620571707"}]},"ts":"1733620571707"} 2024-12-08T01:16:11,709 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T01:16:11,709 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:16:11,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-12-08T01:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T01:16:12,693 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:12,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,694 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc4905a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,696 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559f22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:16:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:16:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3181bcf4 2024-12-08T01:16:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:16:12,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:16:12,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b0d7b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fa842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,706 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19fe7f1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1129c3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256be925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e188dff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,711 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23418088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5509bf73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,714 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38212a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c34fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b44d419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e275af7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,721 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f860850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8a6c2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe7567d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a6ace6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,731 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b08f6eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,732 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3496050a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:12,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:12,734 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44355bac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:12,735 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:12,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308dc5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:12,736 DEBUG [Thread-1895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,736 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:12,737 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,737 DEBUG [Thread-1906 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-08T01:16:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T01:16:12,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:12,742 DEBUG [Thread-1897 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,742 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,743 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:12,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,743 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,744 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,745 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,746 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:12,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:12,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:16:12,754 DEBUG [Thread-1908 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:12,755 DEBUG [Thread-1908 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2] 2024-12-08T01:16:12,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/f52e08b272764743ac7104f5928f238b is 50, key is test_row_0/A:col10/1733620572749/Put/seqid=0 2024-12-08T01:16:12,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742242_1418 (size=12001) 2024-12-08T01:16:12,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/f52e08b272764743ac7104f5928f238b 2024-12-08T01:16:12,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620572782, max timestamp=1733620572798Num uniques 859; 2024-12-08T01:16:12,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620572782, max timestamp=1733620572798Num uniques 859; 2024-12-08T01:16:12,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:12,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:12,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620572782, max timestamp=1733620572798Num uniques 859; 2024-12-08T01:16:12,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:12,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c406484ab2db4c9e93837e3b7cb98609 is 50, key is test_row_0/B:col10/1733620572749/Put/seqid=0 2024-12-08T01:16:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742243_1419 (size=12001) 2024-12-08T01:16:12,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c406484ab2db4c9e93837e3b7cb98609 2024-12-08T01:16:12,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620632841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620632843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620632843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T01:16:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620632845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620632845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/3210f03324a0433993c93b643fce4202 is 50, key is test_row_0/C:col10/1733620572749/Put/seqid=0 2024-12-08T01:16:12,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742244_1420 (size=12001) 2024-12-08T01:16:12,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T01:16:12,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:12,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:12,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:12,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:12,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:12,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620632962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620632962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620632962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620632963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620632963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T01:16:13,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:13,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:13,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:13,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:13,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T01:16:13,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620633172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620633172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620633172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620633172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620633173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T01:16:13,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:13,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:13,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:13,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:13,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/3210f03324a0433993c93b643fce4202 2024-12-08T01:16:13,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/f52e08b272764743ac7104f5928f238b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/f52e08b272764743ac7104f5928f238b 2024-12-08T01:16:13,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/f52e08b272764743ac7104f5928f238b, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T01:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c406484ab2db4c9e93837e3b7cb98609 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c406484ab2db4c9e93837e3b7cb98609 2024-12-08T01:16:13,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c406484ab2db4c9e93837e3b7cb98609, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T01:16:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/3210f03324a0433993c93b643fce4202 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/3210f03324a0433993c93b643fce4202 2024-12-08T01:16:13,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/3210f03324a0433993c93b643fce4202, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T01:16:13,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 192cb468141f6e111e56d290a9940203 in 548ms, sequenceid=12, compaction requested=false 2024-12-08T01:16:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T01:16:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:13,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T01:16:13,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:13,357 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-08T01:16:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:13,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/ee7e6917e8f14427b7ede446ff0c1361 is 50, key is test_row_0/A:col10/1733620572842/Put/seqid=0 2024-12-08T01:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T01:16:13,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742245_1421 (size=12001) 2024-12-08T01:16:13,375 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/ee7e6917e8f14427b7ede446ff0c1361 2024-12-08T01:16:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6cb62d19b0a144b1a0d5720272436457 is 50, key is test_row_0/B:col10/1733620572842/Put/seqid=0 2024-12-08T01:16:13,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742246_1422 (size=12001) 2024-12-08T01:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:13,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620633505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620633504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620633512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620633516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620633517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620633622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620633622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620633632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620633632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620633633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,807 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6cb62d19b0a144b1a0d5720272436457 2024-12-08T01:16:13,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5ee0de54c07546739ce9edc3123c44d8 is 50, key is test_row_0/C:col10/1733620572842/Put/seqid=0 2024-12-08T01:16:13,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742247_1423 (size=12001) 2024-12-08T01:16:13,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620633842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620633842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620633851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620633851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620633852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T01:16:14,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620634151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620634151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620634162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620634162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620634162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,218 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5ee0de54c07546739ce9edc3123c44d8 2024-12-08T01:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/ee7e6917e8f14427b7ede446ff0c1361 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ee7e6917e8f14427b7ede446ff0c1361 2024-12-08T01:16:14,228 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ee7e6917e8f14427b7ede446ff0c1361, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T01:16:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6cb62d19b0a144b1a0d5720272436457 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6cb62d19b0a144b1a0d5720272436457 2024-12-08T01:16:14,232 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6cb62d19b0a144b1a0d5720272436457, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T01:16:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5ee0de54c07546739ce9edc3123c44d8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5ee0de54c07546739ce9edc3123c44d8 2024-12-08T01:16:14,237 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5ee0de54c07546739ce9edc3123c44d8, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T01:16:14,237 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 192cb468141f6e111e56d290a9940203 in 880ms, sequenceid=47, compaction requested=false 2024-12-08T01:16:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-08T01:16:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-08T01:16:14,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-08T01:16:14,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-12-08T01:16:14,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5030 sec 2024-12-08T01:16:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:14,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T01:16:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:14,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/fd221243e24f45c2baab84c285941830 is 50, key is test_row_0/A:col10/1733620574662/Put/seqid=0 2024-12-08T01:16:14,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742248_1424 (size=12001) 2024-12-08T01:16:14,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/fd221243e24f45c2baab84c285941830 2024-12-08T01:16:14,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/8e0e1a009e0e4103bab2729bbe47be6f is 50, key is test_row_0/B:col10/1733620574662/Put/seqid=0 2024-12-08T01:16:14,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742249_1425 (size=12001) 2024-12-08T01:16:14,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/8e0e1a009e0e4103bab2729bbe47be6f 2024-12-08T01:16:14,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f02df9118b0744d4aafbea30ab32b3a6 is 50, key is test_row_0/C:col10/1733620574662/Put/seqid=0 2024-12-08T01:16:14,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742250_1426 (size=12001) 2024-12-08T01:16:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620634723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620634731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620634733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620634735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620634734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620634842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620634842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620634851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620634852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620634852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T01:16:14,892 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:14,893 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-08T01:16:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T01:16:14,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:14,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:14,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T01:16:15,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T01:16:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:15,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:15,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620635051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620635051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620635062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620635062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620635062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f02df9118b0744d4aafbea30ab32b3a6 2024-12-08T01:16:15,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/fd221243e24f45c2baab84c285941830 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/fd221243e24f45c2baab84c285941830 2024-12-08T01:16:15,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/fd221243e24f45c2baab84c285941830, entries=150, sequenceid=59, filesize=11.7 K 2024-12-08T01:16:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/8e0e1a009e0e4103bab2729bbe47be6f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/8e0e1a009e0e4103bab2729bbe47be6f 2024-12-08T01:16:15,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/8e0e1a009e0e4103bab2729bbe47be6f, entries=150, sequenceid=59, filesize=11.7 K 2024-12-08T01:16:15,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f02df9118b0744d4aafbea30ab32b3a6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f02df9118b0744d4aafbea30ab32b3a6 2024-12-08T01:16:15,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f02df9118b0744d4aafbea30ab32b3a6, entries=150, sequenceid=59, filesize=11.7 K 2024-12-08T01:16:15,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 192cb468141f6e111e56d290a9940203 in 463ms, sequenceid=59, compaction requested=true 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:15,126 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:15,126 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:15,127 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:15,127 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:15,127 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:15,127 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/f52e08b272764743ac7104f5928f238b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ee7e6917e8f14427b7ede446ff0c1361, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/fd221243e24f45c2baab84c285941830] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.2 K 2024-12-08T01:16:15,127 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:15,127 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:15,127 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:15,127 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c406484ab2db4c9e93837e3b7cb98609, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6cb62d19b0a144b1a0d5720272436457, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/8e0e1a009e0e4103bab2729bbe47be6f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.2 K 2024-12-08T01:16:15,129 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52e08b272764743ac7104f5928f238b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733620572741 2024-12-08T01:16:15,129 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c406484ab2db4c9e93837e3b7cb98609, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733620572741 2024-12-08T01:16:15,129 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb62d19b0a144b1a0d5720272436457, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733620572833 2024-12-08T01:16:15,129 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee7e6917e8f14427b7ede446ff0c1361, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733620572833 2024-12-08T01:16:15,129 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0e1a009e0e4103bab2729bbe47be6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733620573511 2024-12-08T01:16:15,130 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd221243e24f45c2baab84c285941830, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733620573511 2024-12-08T01:16:15,137 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:15,137 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/5894345109ba4933b268b781990f731b is 50, key is test_row_0/A:col10/1733620574662/Put/seqid=0 2024-12-08T01:16:15,143 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:15,143 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/e46ab0b8eceb4a3fa8902e3602111ff3 is 50, key is test_row_0/B:col10/1733620574662/Put/seqid=0 2024-12-08T01:16:15,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742251_1427 (size=12104) 2024-12-08T01:16:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742252_1428 (size=12104) 2024-12-08T01:16:15,153 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/e46ab0b8eceb4a3fa8902e3602111ff3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e46ab0b8eceb4a3fa8902e3602111ff3 2024-12-08T01:16:15,156 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into e46ab0b8eceb4a3fa8902e3602111ff3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:15,156 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:15,156 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=13, startTime=1733620575126; duration=0sec 2024-12-08T01:16:15,156 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:15,156 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:15,156 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:15,157 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:15,157 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:15,157 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:15,157 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/3210f03324a0433993c93b643fce4202, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5ee0de54c07546739ce9edc3123c44d8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f02df9118b0744d4aafbea30ab32b3a6] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.2 K 2024-12-08T01:16:15,158 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3210f03324a0433993c93b643fce4202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733620572741 2024-12-08T01:16:15,158 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee0de54c07546739ce9edc3123c44d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733620572833 2024-12-08T01:16:15,158 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f02df9118b0744d4aafbea30ab32b3a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733620573511 2024-12-08T01:16:15,162 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:15,163 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5b42c44dd25b4c81b0f0d6a6256e84b6 is 50, key is test_row_0/C:col10/1733620574662/Put/seqid=0 2024-12-08T01:16:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742253_1429 (size=12104) 2024-12-08T01:16:15,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T01:16:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:15,201 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:15,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/3b1a8395b7e2459ebe83575da89ff00f is 50, key is test_row_0/A:col10/1733620574731/Put/seqid=0 2024-12-08T01:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T01:16:15,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742254_1430 (size=12001) 2024-12-08T01:16:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:15,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620635382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620635382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620635386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620635388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620635389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620635502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620635502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620635502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T01:16:15,558 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/5894345109ba4933b268b781990f731b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5894345109ba4933b268b781990f731b 2024-12-08T01:16:15,563 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into 5894345109ba4933b268b781990f731b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:15,563 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:15,563 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=13, startTime=1733620575126; duration=0sec 2024-12-08T01:16:15,563 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:15,563 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:15,574 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5b42c44dd25b4c81b0f0d6a6256e84b6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5b42c44dd25b4c81b0f0d6a6256e84b6 2024-12-08T01:16:15,577 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into 5b42c44dd25b4c81b0f0d6a6256e84b6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:15,577 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:15,578 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=13, startTime=1733620575126; duration=0sec 2024-12-08T01:16:15,578 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:15,578 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:15,613 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/3b1a8395b7e2459ebe83575da89ff00f 2024-12-08T01:16:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/1282c0efc435449d82da822d45b7b329 is 50, key is test_row_0/B:col10/1733620574731/Put/seqid=0 2024-12-08T01:16:15,619 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:16:15,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742255_1431 (size=12001) 2024-12-08T01:16:15,636 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/1282c0efc435449d82da822d45b7b329 2024-12-08T01:16:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/0ca77a5c426945268d0c3c2d00ac9048 is 50, key is test_row_0/C:col10/1733620574731/Put/seqid=0 2024-12-08T01:16:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742256_1432 (size=12001) 2024-12-08T01:16:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620635712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620635712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620635722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620635891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:15,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620635902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620636022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620636022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T01:16:16,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620636032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,050 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/0ca77a5c426945268d0c3c2d00ac9048 2024-12-08T01:16:16,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/3b1a8395b7e2459ebe83575da89ff00f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/3b1a8395b7e2459ebe83575da89ff00f 2024-12-08T01:16:16,055 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/3b1a8395b7e2459ebe83575da89ff00f, entries=150, sequenceid=80, filesize=11.7 K 2024-12-08T01:16:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/1282c0efc435449d82da822d45b7b329 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1282c0efc435449d82da822d45b7b329 2024-12-08T01:16:16,059 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1282c0efc435449d82da822d45b7b329, entries=150, sequenceid=80, filesize=11.7 K 2024-12-08T01:16:16,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/0ca77a5c426945268d0c3c2d00ac9048 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/0ca77a5c426945268d0c3c2d00ac9048 2024-12-08T01:16:16,063 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/0ca77a5c426945268d0c3c2d00ac9048, entries=150, sequenceid=80, filesize=11.7 K 2024-12-08T01:16:16,064 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 192cb468141f6e111e56d290a9940203 in 863ms, sequenceid=80, compaction requested=false 2024-12-08T01:16:16,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:16,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:16,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-08T01:16:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-08T01:16:16,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-08T01:16:16,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-12-08T01:16:16,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1740 sec 2024-12-08T01:16:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:16,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T01:16:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:16,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:16,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:16,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:16,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:16,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:16,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/9deb7e8a48444b8abdefff2da0891fbc is 50, key is test_row_0/A:col10/1733620575380/Put/seqid=0 2024-12-08T01:16:16,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742257_1433 (size=14341) 2024-12-08T01:16:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620636603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620636612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620636613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620636722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620636722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620636732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620636916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620636916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,922 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,923 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620636933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620636942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/9deb7e8a48444b8abdefff2da0891fbc 2024-12-08T01:16:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c69e29a06f2c4f148f8f7b13722ce6b9 is 50, key is test_row_0/B:col10/1733620575380/Put/seqid=0 2024-12-08T01:16:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620636952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:16,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742258_1434 (size=12001) 2024-12-08T01:16:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T01:16:17,042 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:17,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-08T01:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T01:16:17,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:17,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:17,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T01:16:17,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T01:16:17,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620637252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620637252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620637261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:17,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T01:16:17,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c69e29a06f2c4f148f8f7b13722ce6b9 2024-12-08T01:16:17,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/1bbb1468fbfd458e8ff06fad523b8ef6 is 50, key is test_row_0/C:col10/1733620575380/Put/seqid=0 2024-12-08T01:16:17,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742259_1435 (size=12001) 2024-12-08T01:16:17,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/1bbb1468fbfd458e8ff06fad523b8ef6 2024-12-08T01:16:17,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/9deb7e8a48444b8abdefff2da0891fbc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/9deb7e8a48444b8abdefff2da0891fbc 2024-12-08T01:16:17,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/9deb7e8a48444b8abdefff2da0891fbc, entries=200, sequenceid=95, filesize=14.0 K 2024-12-08T01:16:17,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c69e29a06f2c4f148f8f7b13722ce6b9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c69e29a06f2c4f148f8f7b13722ce6b9 2024-12-08T01:16:17,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c69e29a06f2c4f148f8f7b13722ce6b9, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T01:16:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/1bbb1468fbfd458e8ff06fad523b8ef6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1bbb1468fbfd458e8ff06fad523b8ef6 2024-12-08T01:16:17,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1bbb1468fbfd458e8ff06fad523b8ef6, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T01:16:17,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 192cb468141f6e111e56d290a9940203 in 860ms, sequenceid=95, compaction requested=true 2024-12-08T01:16:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:17,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:17,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:17,394 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:17,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:17,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:17,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:17,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:16:17,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:17,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:17,394 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:17,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:17,394 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:17,395 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,395 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,395 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e46ab0b8eceb4a3fa8902e3602111ff3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1282c0efc435449d82da822d45b7b329, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c69e29a06f2c4f148f8f7b13722ce6b9] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.3 K 2024-12-08T01:16:17,395 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5894345109ba4933b268b781990f731b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/3b1a8395b7e2459ebe83575da89ff00f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/9deb7e8a48444b8abdefff2da0891fbc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=37.5 K 2024-12-08T01:16:17,395 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5894345109ba4933b268b781990f731b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733620573511 2024-12-08T01:16:17,395 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting e46ab0b8eceb4a3fa8902e3602111ff3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733620573511 2024-12-08T01:16:17,395 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1a8395b7e2459ebe83575da89ff00f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733620574731 2024-12-08T01:16:17,395 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1282c0efc435449d82da822d45b7b329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733620574731 2024-12-08T01:16:17,395 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c69e29a06f2c4f148f8f7b13722ce6b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620575380 2024-12-08T01:16:17,395 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9deb7e8a48444b8abdefff2da0891fbc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620575370 2024-12-08T01:16:17,414 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:17,414 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/e5a9e41acca8425295f81c7fc6f34d0e is 50, key is test_row_0/B:col10/1733620575380/Put/seqid=0 2024-12-08T01:16:17,423 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#370 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:17,424 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/e4ae514423ad4ca4ab3bb1f2345eb429 is 50, key is test_row_0/A:col10/1733620575380/Put/seqid=0 2024-12-08T01:16:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742260_1436 (size=12207) 2024-12-08T01:16:17,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742261_1437 (size=12207) 2024-12-08T01:16:17,458 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/e5a9e41acca8425295f81c7fc6f34d0e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e5a9e41acca8425295f81c7fc6f34d0e 2024-12-08T01:16:17,461 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/e4ae514423ad4ca4ab3bb1f2345eb429 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e4ae514423ad4ca4ab3bb1f2345eb429 2024-12-08T01:16:17,463 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into e5a9e41acca8425295f81c7fc6f34d0e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:17,463 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:17,463 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=13, startTime=1733620577394; duration=0sec 2024-12-08T01:16:17,463 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:17,463 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:17,463 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:17,464 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:17,464 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:17,464 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,464 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5b42c44dd25b4c81b0f0d6a6256e84b6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/0ca77a5c426945268d0c3c2d00ac9048, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1bbb1468fbfd458e8ff06fad523b8ef6] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.3 K 2024-12-08T01:16:17,465 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b42c44dd25b4c81b0f0d6a6256e84b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733620573511 2024-12-08T01:16:17,465 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca77a5c426945268d0c3c2d00ac9048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733620574731 2024-12-08T01:16:17,465 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bbb1468fbfd458e8ff06fad523b8ef6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620575380 2024-12-08T01:16:17,469 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into e4ae514423ad4ca4ab3bb1f2345eb429(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:17,469 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:17,469 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=13, startTime=1733620577394; duration=0sec 2024-12-08T01:16:17,469 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:17,469 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:17,474 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:17,474 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/b04bc96506294de78382b5d675028477 is 50, key is test_row_0/C:col10/1733620575380/Put/seqid=0 2024-12-08T01:16:17,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:17,503 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:17,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742262_1438 (size=12207) 2024-12-08T01:16:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/7460d59bd81544249972bb2f7a5f7482 is 50, key is test_row_0/A:col10/1733620576581/Put/seqid=0 2024-12-08T01:16:17,521 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/b04bc96506294de78382b5d675028477 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/b04bc96506294de78382b5d675028477 2024-12-08T01:16:17,534 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into b04bc96506294de78382b5d675028477(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:17,534 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:17,534 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=13, startTime=1733620577394; duration=0sec 2024-12-08T01:16:17,534 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:17,534 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:17,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742263_1439 (size=12001) 2024-12-08T01:16:17,554 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/7460d59bd81544249972bb2f7a5f7482 2024-12-08T01:16:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/af6a42844aeb42a4b9249b911e40212a is 50, key is test_row_0/B:col10/1733620576581/Put/seqid=0 2024-12-08T01:16:17,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742264_1440 (size=12001) 2024-12-08T01:16:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T01:16:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:17,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:17,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620637783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620637785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620637784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620637901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620637902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620637902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:17,979 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/af6a42844aeb42a4b9249b911e40212a 2024-12-08T01:16:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/87bad5c84925402999b0b078ee9ede60 is 50, key is test_row_0/C:col10/1733620576581/Put/seqid=0 2024-12-08T01:16:17,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742265_1441 (size=12001) 2024-12-08T01:16:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620638112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620638112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620638114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T01:16:18,399 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/87bad5c84925402999b0b078ee9ede60 2024-12-08T01:16:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/7460d59bd81544249972bb2f7a5f7482 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7460d59bd81544249972bb2f7a5f7482 2024-12-08T01:16:18,405 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7460d59bd81544249972bb2f7a5f7482, entries=150, sequenceid=117, filesize=11.7 K 2024-12-08T01:16:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/af6a42844aeb42a4b9249b911e40212a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af6a42844aeb42a4b9249b911e40212a 2024-12-08T01:16:18,417 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af6a42844aeb42a4b9249b911e40212a, entries=150, sequenceid=117, filesize=11.7 K 2024-12-08T01:16:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/87bad5c84925402999b0b078ee9ede60 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/87bad5c84925402999b0b078ee9ede60 2024-12-08T01:16:18,423 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/87bad5c84925402999b0b078ee9ede60, entries=150, sequenceid=117, filesize=11.7 K 2024-12-08T01:16:18,423 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 192cb468141f6e111e56d290a9940203 in 920ms, sequenceid=117, compaction requested=false 2024-12-08T01:16:18,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:18,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-08T01:16:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-08T01:16:18,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-08T01:16:18,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-12-08T01:16:18,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3840 sec 2024-12-08T01:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:18,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:16:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:18,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/4944ef947f3c4d7f9af694f882c32261 is 50, key is test_row_0/A:col10/1733620578427/Put/seqid=0 2024-12-08T01:16:18,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742266_1442 (size=12151) 2024-12-08T01:16:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620638474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620638475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620638483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620638592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620638592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620638602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620638801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620638801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620638822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/4944ef947f3c4d7f9af694f882c32261 2024-12-08T01:16:18,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6810fc8ed25441c7a526cc2048feb237 is 50, key is test_row_0/B:col10/1733620578427/Put/seqid=0 2024-12-08T01:16:18,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742267_1443 (size=12151) 2024-12-08T01:16:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620638932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620638932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:18,935 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:18,935 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620639112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620639112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620639132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T01:16:19,202 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:19,202 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-08T01:16:19,206 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T01:16:19,207 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:19,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:19,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6810fc8ed25441c7a526cc2048feb237 2024-12-08T01:16:19,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/562c24e2670548659396c9855accff6b is 50, key is test_row_0/C:col10/1733620578427/Put/seqid=0 2024-12-08T01:16:19,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742268_1444 (size=12151) 2024-12-08T01:16:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T01:16:19,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T01:16:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T01:16:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T01:16:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620639622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620639622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620639652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:19,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/562c24e2670548659396c9855accff6b 2024-12-08T01:16:19,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T01:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:19,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/4944ef947f3c4d7f9af694f882c32261 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/4944ef947f3c4d7f9af694f882c32261 2024-12-08T01:16:19,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/4944ef947f3c4d7f9af694f882c32261, entries=150, sequenceid=133, filesize=11.9 K 2024-12-08T01:16:19,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6810fc8ed25441c7a526cc2048feb237 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6810fc8ed25441c7a526cc2048feb237 2024-12-08T01:16:19,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6810fc8ed25441c7a526cc2048feb237, entries=150, sequenceid=133, filesize=11.9 K 2024-12-08T01:16:19,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/562c24e2670548659396c9855accff6b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/562c24e2670548659396c9855accff6b 2024-12-08T01:16:19,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/562c24e2670548659396c9855accff6b, entries=150, sequenceid=133, filesize=11.9 K 2024-12-08T01:16:19,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 192cb468141f6e111e56d290a9940203 in 1256ms, sequenceid=133, compaction requested=true 2024-12-08T01:16:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:19,684 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:16:19,685 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:19,685 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:19,685 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:19,685 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,685 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e4ae514423ad4ca4ab3bb1f2345eb429, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7460d59bd81544249972bb2f7a5f7482, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/4944ef947f3c4d7f9af694f882c32261] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.5 K 2024-12-08T01:16:19,687 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ae514423ad4ca4ab3bb1f2345eb429, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620575380 2024-12-08T01:16:19,687 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7460d59bd81544249972bb2f7a5f7482, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733620576581 2024-12-08T01:16:19,687 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:19,687 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:19,688 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,688 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e5a9e41acca8425295f81c7fc6f34d0e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af6a42844aeb42a4b9249b911e40212a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6810fc8ed25441c7a526cc2048feb237] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.5 K 2024-12-08T01:16:19,688 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4944ef947f3c4d7f9af694f882c32261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733620578426 2024-12-08T01:16:19,688 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a9e41acca8425295f81c7fc6f34d0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620575380 2024-12-08T01:16:19,688 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting af6a42844aeb42a4b9249b911e40212a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733620576581 2024-12-08T01:16:19,688 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6810fc8ed25441c7a526cc2048feb237, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733620578426 2024-12-08T01:16:19,694 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:19,695 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/c350f11333ee4ebaa0907e421973c364 is 50, key is test_row_0/A:col10/1733620578427/Put/seqid=0 2024-12-08T01:16:19,697 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#379 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:19,697 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/fe9d2bf177b04254acfc380a1948f47c is 50, key is test_row_0/B:col10/1733620578427/Put/seqid=0 2024-12-08T01:16:19,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742269_1445 (size=12459) 2024-12-08T01:16:19,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742270_1446 (size=12459) 2024-12-08T01:16:19,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T01:16:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:19,819 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:19,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/31bebb9437364e2fbb5b60a6dea3e854 is 50, key is test_row_0/A:col10/1733620578455/Put/seqid=0 2024-12-08T01:16:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742271_1447 (size=12151) 2024-12-08T01:16:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T01:16:20,110 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/c350f11333ee4ebaa0907e421973c364 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/c350f11333ee4ebaa0907e421973c364 2024-12-08T01:16:20,115 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into c350f11333ee4ebaa0907e421973c364(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:20,115 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:20,115 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=13, startTime=1733620579684; duration=0sec 2024-12-08T01:16:20,115 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:20,115 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:20,115 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:20,117 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/fe9d2bf177b04254acfc380a1948f47c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/fe9d2bf177b04254acfc380a1948f47c 2024-12-08T01:16:20,117 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:20,117 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:20,117 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,117 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/b04bc96506294de78382b5d675028477, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/87bad5c84925402999b0b078ee9ede60, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/562c24e2670548659396c9855accff6b] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.5 K 2024-12-08T01:16:20,118 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04bc96506294de78382b5d675028477, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733620575380 2024-12-08T01:16:20,118 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87bad5c84925402999b0b078ee9ede60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733620576581 2024-12-08T01:16:20,118 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562c24e2670548659396c9855accff6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733620578426 2024-12-08T01:16:20,121 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into fe9d2bf177b04254acfc380a1948f47c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:20,121 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:20,121 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=13, startTime=1733620579684; duration=0sec 2024-12-08T01:16:20,122 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:20,122 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:20,127 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:20,127 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/a8efdc8050284a6c9d7bf87ae282886a is 50, key is test_row_0/C:col10/1733620578427/Put/seqid=0 2024-12-08T01:16:20,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742272_1448 (size=12459) 2024-12-08T01:16:20,230 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/31bebb9437364e2fbb5b60a6dea3e854 2024-12-08T01:16:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6161365aed234e47bc15944afaf57c55 is 50, key is test_row_0/B:col10/1733620578455/Put/seqid=0 2024-12-08T01:16:20,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742273_1449 (size=12151) 2024-12-08T01:16:20,240 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6161365aed234e47bc15944afaf57c55 2024-12-08T01:16:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/a4958a21b74e49f39833ad4429713c71 is 50, key is test_row_0/C:col10/1733620578455/Put/seqid=0 2024-12-08T01:16:20,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742274_1450 (size=12151) 2024-12-08T01:16:20,259 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/a4958a21b74e49f39833ad4429713c71 2024-12-08T01:16:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/31bebb9437364e2fbb5b60a6dea3e854 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/31bebb9437364e2fbb5b60a6dea3e854 2024-12-08T01:16:20,267 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/31bebb9437364e2fbb5b60a6dea3e854, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T01:16:20,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6161365aed234e47bc15944afaf57c55 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6161365aed234e47bc15944afaf57c55 2024-12-08T01:16:20,270 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6161365aed234e47bc15944afaf57c55, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T01:16:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/a4958a21b74e49f39833ad4429713c71 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a4958a21b74e49f39833ad4429713c71 2024-12-08T01:16:20,276 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a4958a21b74e49f39833ad4429713c71, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T01:16:20,277 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 192cb468141f6e111e56d290a9940203 in 459ms, sequenceid=150, compaction requested=false 2024-12-08T01:16:20,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:20,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-08T01:16:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-08T01:16:20,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-08T01:16:20,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0720 sec 2024-12-08T01:16:20,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0790 sec 2024-12-08T01:16:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T01:16:20,342 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:20,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-08T01:16:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T01:16:20,344 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:20,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:20,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T01:16:20,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T01:16:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-08T01:16:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-08T01:16:20,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-08T01:16:20,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-08T01:16:20,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 158 msec 2024-12-08T01:16:20,540 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/a8efdc8050284a6c9d7bf87ae282886a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a8efdc8050284a6c9d7bf87ae282886a 2024-12-08T01:16:20,544 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into a8efdc8050284a6c9d7bf87ae282886a(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:20,544 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:20,544 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=13, startTime=1733620579684; duration=0sec 2024-12-08T01:16:20,544 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:20,544 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T01:16:20,662 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:20,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-08T01:16:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:20,664 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:20,665 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:20,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:20,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:16:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:20,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/ed45390e4870499aa1a733f2e967f958 is 50, key is test_row_0/A:col10/1733620580650/Put/seqid=0 2024-12-08T01:16:20,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742275_1451 (size=12151) 2024-12-08T01:16:20,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620580700, max timestamp=1733620580717Num uniques 859; 2024-12-08T01:16:20,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:16:20,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620580700, max timestamp=1733620580717Num uniques 859; 2024-12-08T01:16:20,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:16:20,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733620580700, max timestamp=1733620580717Num uniques 859; 2024-12-08T01:16:20,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T01:16:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620640755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:20,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620640761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:20,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620640762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:20,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620640872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620640872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:20,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620640872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:20,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:20,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:21,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/ed45390e4870499aa1a733f2e967f958 2024-12-08T01:16:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620641082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620641082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620641082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/a4013488b4d94901b7c6f7f5fc54b9a2 is 50, key is test_row_0/B:col10/1733620580650/Put/seqid=0 2024-12-08T01:16:21,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742276_1452 (size=12151) 2024-12-08T01:16:21,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:21,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:21,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:21,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620641391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620641391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620641402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:21,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:21,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/a4013488b4d94901b7c6f7f5fc54b9a2 2024-12-08T01:16:21,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/9653d01a316a4648a88f851caf53d439 is 50, key is test_row_0/C:col10/1733620580650/Put/seqid=0 2024-12-08T01:16:21,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742277_1453 (size=12151) 2024-12-08T01:16:21,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:21,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:21,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:21,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:21,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/9653d01a316a4648a88f851caf53d439 2024-12-08T01:16:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/ed45390e4870499aa1a733f2e967f958 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ed45390e4870499aa1a733f2e967f958 2024-12-08T01:16:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620641902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ed45390e4870499aa1a733f2e967f958, entries=150, sequenceid=165, filesize=11.9 K 2024-12-08T01:16:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/a4013488b4d94901b7c6f7f5fc54b9a2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/a4013488b4d94901b7c6f7f5fc54b9a2 2024-12-08T01:16:21,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/a4013488b4d94901b7c6f7f5fc54b9a2, entries=150, sequenceid=165, filesize=11.9 K 2024-12-08T01:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/9653d01a316a4648a88f851caf53d439 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9653d01a316a4648a88f851caf53d439 2024-12-08T01:16:21,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9653d01a316a4648a88f851caf53d439, entries=150, sequenceid=165, filesize=11.9 K 2024-12-08T01:16:21,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 192cb468141f6e111e56d290a9940203 in 1241ms, sequenceid=165, compaction requested=true 2024-12-08T01:16:21,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:21,921 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:21,921 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:21,924 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:21,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T01:16:21,924 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:21,924 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,924 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/c350f11333ee4ebaa0907e421973c364, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/31bebb9437364e2fbb5b60a6dea3e854, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ed45390e4870499aa1a733f2e967f958] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.9 K 2024-12-08T01:16:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:21,924 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:21,924 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:21,924 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,924 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c350f11333ee4ebaa0907e421973c364, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733620578426 2024-12-08T01:16:21,924 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/fe9d2bf177b04254acfc380a1948f47c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6161365aed234e47bc15944afaf57c55, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/a4013488b4d94901b7c6f7f5fc54b9a2] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.9 K 2024-12-08T01:16:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:21,924 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31bebb9437364e2fbb5b60a6dea3e854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733620578455 2024-12-08T01:16:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:21,925 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9d2bf177b04254acfc380a1948f47c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733620578426 2024-12-08T01:16:21,925 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6161365aed234e47bc15944afaf57c55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733620578455 2024-12-08T01:16:21,925 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed45390e4870499aa1a733f2e967f958, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733620580645 2024-12-08T01:16:21,925 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a4013488b4d94901b7c6f7f5fc54b9a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733620580645 2024-12-08T01:16:21,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/216c60830e38496f830c4ad52a5e993d is 50, key is test_row_0/A:col10/1733620580754/Put/seqid=0 2024-12-08T01:16:21,934 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:21,935 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/1aa0c8265c5947cb849ab303e94b45ae is 50, key is test_row_0/B:col10/1733620580650/Put/seqid=0 2024-12-08T01:16:21,943 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:21,943 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/27e110df0a0c473db2e7eac566539814 is 50, key is test_row_0/A:col10/1733620580650/Put/seqid=0 2024-12-08T01:16:21,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742279_1455 (size=12561) 2024-12-08T01:16:21,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742278_1454 (size=14541) 2024-12-08T01:16:21,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742280_1456 (size=12561) 2024-12-08T01:16:21,958 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/27e110df0a0c473db2e7eac566539814 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/27e110df0a0c473db2e7eac566539814 2024-12-08T01:16:21,965 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into 27e110df0a0c473db2e7eac566539814(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:21,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:21,965 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=13, startTime=1733620581921; duration=0sec 2024-12-08T01:16:21,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:21,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:21,965 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:21,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620641959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,966 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:21,966 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:21,967 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:21,967 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a8efdc8050284a6c9d7bf87ae282886a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a4958a21b74e49f39833ad4429713c71, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9653d01a316a4648a88f851caf53d439] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=35.9 K 2024-12-08T01:16:21,967 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8efdc8050284a6c9d7bf87ae282886a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733620578426 2024-12-08T01:16:21,967 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4958a21b74e49f39833ad4429713c71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733620578455 2024-12-08T01:16:21,967 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9653d01a316a4648a88f851caf53d439, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733620580645 2024-12-08T01:16:21,972 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#390 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:21,972 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/1393c0f222ab462cb05d6efe298c39c8 is 50, key is test_row_0/C:col10/1733620580650/Put/seqid=0 2024-12-08T01:16:21,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742281_1457 (size=12561) 2024-12-08T01:16:21,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620641966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:21,983 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/1393c0f222ab462cb05d6efe298c39c8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1393c0f222ab462cb05d6efe298c39c8 2024-12-08T01:16:21,987 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into 1393c0f222ab462cb05d6efe298c39c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:21,987 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:21,988 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=13, startTime=1733620581921; duration=0sec 2024-12-08T01:16:21,988 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:21,988 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:22,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620642072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:22,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620642082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:22,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620642282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:22,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620642291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:22,348 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/1aa0c8265c5947cb849ab303e94b45ae as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1aa0c8265c5947cb849ab303e94b45ae 2024-12-08T01:16:22,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,352 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into 1aa0c8265c5947cb849ab303e94b45ae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:22,352 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:22,352 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=13, startTime=1733620581921; duration=0sec 2024-12-08T01:16:22,352 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:22,352 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:22,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/216c60830e38496f830c4ad52a5e993d 2024-12-08T01:16:22,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6e229304d1a4499ebe45a8c7dc16a649 is 50, key is test_row_0/B:col10/1733620580754/Put/seqid=0 2024-12-08T01:16:22,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742282_1458 (size=12151) 2024-12-08T01:16:22,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6e229304d1a4499ebe45a8c7dc16a649 2024-12-08T01:16:22,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/ee1417430475475aa48ad1e68998442e is 50, key is test_row_0/C:col10/1733620580754/Put/seqid=0 2024-12-08T01:16:22,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742283_1459 (size=12151) 2024-12-08T01:16:22,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620642592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:22,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620642602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:22,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/ee1417430475475aa48ad1e68998442e 2024-12-08T01:16:22,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/216c60830e38496f830c4ad52a5e993d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/216c60830e38496f830c4ad52a5e993d 2024-12-08T01:16:22,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/216c60830e38496f830c4ad52a5e993d, entries=200, sequenceid=200, filesize=14.2 K 2024-12-08T01:16:22,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6e229304d1a4499ebe45a8c7dc16a649 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6e229304d1a4499ebe45a8c7dc16a649 2024-12-08T01:16:22,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6e229304d1a4499ebe45a8c7dc16a649, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T01:16:22,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/ee1417430475475aa48ad1e68998442e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/ee1417430475475aa48ad1e68998442e 2024-12-08T01:16:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:22,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/ee1417430475475aa48ad1e68998442e, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T01:16:22,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 192cb468141f6e111e56d290a9940203 in 899ms, sequenceid=200, compaction requested=false 2024-12-08T01:16:22,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:22,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:16:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:22,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/93c2bd08d0be46b6a51747e187970cfb is 50, key is test_row_0/A:col10/1733620582924/Put/seqid=0 2024-12-08T01:16:22,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742284_1460 (size=14541) 2024-12-08T01:16:22,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/93c2bd08d0be46b6a51747e187970cfb 2024-12-08T01:16:22,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/78bb9ec6f3b6408289f2e7dc3a268478 is 50, key is test_row_0/B:col10/1733620582924/Put/seqid=0 2024-12-08T01:16:22,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:22,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742285_1461 (size=12151) 2024-12-08T01:16:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620643000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620582996, max timestamp=1733620583009Num uniques 859; 2024-12-08T01:16:23,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:23,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620582996, max timestamp=1733620583009Num uniques 859; 2024-12-08T01:16:23,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:23,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733620582996, max timestamp=1733620583009Num uniques 859; 2024-12-08T01:16:23,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:23,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620643060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620643070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620643102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620643122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620643122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620643182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620643182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:23,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:23,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620643342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/78bb9ec6f3b6408289f2e7dc3a268478 2024-12-08T01:16:23,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/886cf571069f4f68ad2ac2de964bd44e is 50, key is test_row_0/C:col10/1733620582924/Put/seqid=0 2024-12-08T01:16:23,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742286_1462 (size=12151) 2024-12-08T01:16:23,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620643392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620643392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:23,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:23,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620643652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620643701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620643701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:23,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:23,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/886cf571069f4f68ad2ac2de964bd44e 2024-12-08T01:16:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/93c2bd08d0be46b6a51747e187970cfb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/93c2bd08d0be46b6a51747e187970cfb 2024-12-08T01:16:23,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/93c2bd08d0be46b6a51747e187970cfb, entries=200, sequenceid=214, filesize=14.2 K 2024-12-08T01:16:23,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/78bb9ec6f3b6408289f2e7dc3a268478 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/78bb9ec6f3b6408289f2e7dc3a268478 2024-12-08T01:16:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/78bb9ec6f3b6408289f2e7dc3a268478, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T01:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/886cf571069f4f68ad2ac2de964bd44e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/886cf571069f4f68ad2ac2de964bd44e 2024-12-08T01:16:23,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/886cf571069f4f68ad2ac2de964bd44e, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T01:16:23,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 192cb468141f6e111e56d290a9940203 in 872ms, sequenceid=214, compaction requested=true 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:23,798 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:23,798 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:23,798 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:23,798 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:23,799 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,799 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,799 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1aa0c8265c5947cb849ab303e94b45ae, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6e229304d1a4499ebe45a8c7dc16a649, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/78bb9ec6f3b6408289f2e7dc3a268478] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=36.0 K 2024-12-08T01:16:23,799 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/27e110df0a0c473db2e7eac566539814, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/216c60830e38496f830c4ad52a5e993d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/93c2bd08d0be46b6a51747e187970cfb] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=40.7 K 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e110df0a0c473db2e7eac566539814, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733620580645 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aa0c8265c5947cb849ab303e94b45ae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733620580645 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e229304d1a4499ebe45a8c7dc16a649, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733620580754 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216c60830e38496f830c4ad52a5e993d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733620580754 2024-12-08T01:16:23,799 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bb9ec6f3b6408289f2e7dc3a268478, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733620581938 2024-12-08T01:16:23,800 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c2bd08d0be46b6a51747e187970cfb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733620581938 2024-12-08T01:16:23,804 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:23,804 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/cb7fecb3bfff436ab6496a48fb4d5bb9 is 50, key is test_row_0/A:col10/1733620582924/Put/seqid=0 2024-12-08T01:16:23,805 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:23,805 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/9e4216822e6a4ffcb1031bfb8c8ccf6e is 50, key is test_row_0/B:col10/1733620582924/Put/seqid=0 2024-12-08T01:16:23,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742287_1463 (size=12663) 2024-12-08T01:16:23,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742288_1464 (size=12663) 2024-12-08T01:16:23,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T01:16:23,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:23,883 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-08T01:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/dcd15a300e6c431d92e03995e63c1263 is 50, key is test_row_0/A:col10/1733620583057/Put/seqid=0 2024-12-08T01:16:23,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742289_1465 (size=12151) 2024-12-08T01:16:23,899 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/dcd15a300e6c431d92e03995e63c1263 2024-12-08T01:16:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/316cd29a67dc4709b29d7700a416c50f is 50, key is test_row_0/B:col10/1733620583057/Put/seqid=0 2024-12-08T01:16:23,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742290_1466 (size=12151) 2024-12-08T01:16:23,931 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/316cd29a67dc4709b29d7700a416c50f 2024-12-08T01:16:23,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/e5774f8ca15c44e6adede851ee290055 is 50, key is test_row_0/C:col10/1733620583057/Put/seqid=0 2024-12-08T01:16:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742291_1467 (size=12151) 2024-12-08T01:16:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:24,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620644170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620644172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620644179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,215 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/9e4216822e6a4ffcb1031bfb8c8ccf6e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9e4216822e6a4ffcb1031bfb8c8ccf6e 2024-12-08T01:16:24,219 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/cb7fecb3bfff436ab6496a48fb4d5bb9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/cb7fecb3bfff436ab6496a48fb4d5bb9 2024-12-08T01:16:24,220 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into 9e4216822e6a4ffcb1031bfb8c8ccf6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:24,220 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,220 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=13, startTime=1733620583798; duration=0sec 2024-12-08T01:16:24,220 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:24,220 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:24,220 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:24,221 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:24,221 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:24,221 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,221 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1393c0f222ab462cb05d6efe298c39c8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/ee1417430475475aa48ad1e68998442e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/886cf571069f4f68ad2ac2de964bd44e] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=36.0 K 2024-12-08T01:16:24,222 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1393c0f222ab462cb05d6efe298c39c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733620580645 2024-12-08T01:16:24,222 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1417430475475aa48ad1e68998442e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733620580754 2024-12-08T01:16:24,222 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 886cf571069f4f68ad2ac2de964bd44e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733620581938 2024-12-08T01:16:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620644212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,229 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into cb7fecb3bfff436ab6496a48fb4d5bb9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:24,229 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,229 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=13, startTime=1733620583798; duration=0sec 2024-12-08T01:16:24,230 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:24,230 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:24,234 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620644222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,234 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/53d3d1316b294856a552a372c21ee4eb is 50, key is test_row_0/C:col10/1733620582924/Put/seqid=0 2024-12-08T01:16:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742292_1468 (size=12663) 2024-12-08T01:16:24,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620644293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620644302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,375 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/e5774f8ca15c44e6adede851ee290055 2024-12-08T01:16:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/dcd15a300e6c431d92e03995e63c1263 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/dcd15a300e6c431d92e03995e63c1263 2024-12-08T01:16:24,385 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/dcd15a300e6c431d92e03995e63c1263, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T01:16:24,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/316cd29a67dc4709b29d7700a416c50f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/316cd29a67dc4709b29d7700a416c50f 2024-12-08T01:16:24,390 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/316cd29a67dc4709b29d7700a416c50f, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T01:16:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/e5774f8ca15c44e6adede851ee290055 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e5774f8ca15c44e6adede851ee290055 2024-12-08T01:16:24,399 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e5774f8ca15c44e6adede851ee290055, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T01:16:24,400 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 192cb468141f6e111e56d290a9940203 in 517ms, sequenceid=249, compaction requested=false 2024-12-08T01:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-08T01:16:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-08T01:16:24,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-08T01:16:24,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7390 sec 2024-12-08T01:16:24,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.7520 sec 2024-12-08T01:16:24,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T01:16:24,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:24,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:24,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:24,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:24,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:24,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:24,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/caaa9ba6f0414e5485a171afdc4e6453 is 50, key is test_row_0/A:col10/1733620584514/Put/seqid=0 2024-12-08T01:16:24,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742293_1469 (size=14741) 2024-12-08T01:16:24,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/caaa9ba6f0414e5485a171afdc4e6453 2024-12-08T01:16:24,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/7c5797eebbd2452eb1bacd58bd44999c is 50, key is test_row_0/B:col10/1733620584514/Put/seqid=0 2024-12-08T01:16:24,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742294_1470 (size=12301) 2024-12-08T01:16:24,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/7c5797eebbd2452eb1bacd58bd44999c 2024-12-08T01:16:24,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/7bd1f8bd9334441589c95ba4104b200a is 50, key is test_row_0/C:col10/1733620584514/Put/seqid=0 2024-12-08T01:16:24,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742295_1471 (size=12301) 2024-12-08T01:16:24,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/7bd1f8bd9334441589c95ba4104b200a 2024-12-08T01:16:24,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/caaa9ba6f0414e5485a171afdc4e6453 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/caaa9ba6f0414e5485a171afdc4e6453 2024-12-08T01:16:24,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/caaa9ba6f0414e5485a171afdc4e6453, entries=200, sequenceid=263, filesize=14.4 K 2024-12-08T01:16:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/7c5797eebbd2452eb1bacd58bd44999c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/7c5797eebbd2452eb1bacd58bd44999c 2024-12-08T01:16:24,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/7c5797eebbd2452eb1bacd58bd44999c, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:16:24,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/7bd1f8bd9334441589c95ba4104b200a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7bd1f8bd9334441589c95ba4104b200a 2024-12-08T01:16:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620644609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620644610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7bd1f8bd9334441589c95ba4104b200a, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:16:24,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 192cb468141f6e111e56d290a9940203 in 108ms, sequenceid=263, compaction requested=true 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:24,624 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:16:24,625 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:24,625 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:24,625 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,625 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/cb7fecb3bfff436ab6496a48fb4d5bb9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/dcd15a300e6c431d92e03995e63c1263, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/caaa9ba6f0414e5485a171afdc4e6453] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=38.6 K 2024-12-08T01:16:24,626 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7fecb3bfff436ab6496a48fb4d5bb9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733620581938 2024-12-08T01:16:24,626 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcd15a300e6c431d92e03995e63c1263, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733620583057 2024-12-08T01:16:24,626 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting caaa9ba6f0414e5485a171afdc4e6453, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620584164 2024-12-08T01:16:24,647 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:24,647 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/243ba2155a3a4ce6a6eb9c981ca89d22 is 50, key is test_row_0/A:col10/1733620584514/Put/seqid=0 2024-12-08T01:16:24,650 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/53d3d1316b294856a552a372c21ee4eb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/53d3d1316b294856a552a372c21ee4eb 2024-12-08T01:16:24,660 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into 53d3d1316b294856a552a372c21ee4eb(size=12.4 K), total size for store is 36.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:24,660 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,660 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=13, startTime=1733620583798; duration=0sec 2024-12-08T01:16:24,660 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T01:16:24,660 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:24,660 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:24,660 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:24,665 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:24,665 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:24,665 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,665 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9e4216822e6a4ffcb1031bfb8c8ccf6e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/316cd29a67dc4709b29d7700a416c50f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/7c5797eebbd2452eb1bacd58bd44999c] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=36.2 K 2024-12-08T01:16:24,665 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e4216822e6a4ffcb1031bfb8c8ccf6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733620581938 2024-12-08T01:16:24,666 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 316cd29a67dc4709b29d7700a416c50f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733620583057 2024-12-08T01:16:24,666 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c5797eebbd2452eb1bacd58bd44999c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620584164 2024-12-08T01:16:24,679 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:24,679 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/149fb160359c45539682913aeff476a3 is 50, key is test_row_0/B:col10/1733620584514/Put/seqid=0 2024-12-08T01:16:24,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742296_1472 (size=12915) 2024-12-08T01:16:24,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742297_1473 (size=12915) 2024-12-08T01:16:24,726 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/149fb160359c45539682913aeff476a3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/149fb160359c45539682913aeff476a3 2024-12-08T01:16:24,732 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into 149fb160359c45539682913aeff476a3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:24,732 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,732 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=13, startTime=1733620584624; duration=0sec 2024-12-08T01:16:24,732 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:24,732 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:24,732 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:24,733 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:24,733 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:24,733 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,733 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/53d3d1316b294856a552a372c21ee4eb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e5774f8ca15c44e6adede851ee290055, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7bd1f8bd9334441589c95ba4104b200a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=36.2 K 2024-12-08T01:16:24,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620584600, max timestamp=1733620584733Num uniques 859; 2024-12-08T01:16:24,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:24,736 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d3d1316b294856a552a372c21ee4eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733620581938 2024-12-08T01:16:24,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting e5774f8ca15c44e6adede851ee290055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733620583057 2024-12-08T01:16:24,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd1f8bd9334441589c95ba4104b200a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620584164 2024-12-08T01:16:24,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620584600, max timestamp=1733620584736Num uniques 859; 2024-12-08T01:16:24,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:24,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620584600, max timestamp=1733620584733Num uniques 859; 2024-12-08T01:16:24,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:24,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-08T01:16:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:24,755 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:24,755 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/7ab1f179670b4c1fb3ced6ae19ed9968 is 50, key is test_row_0/C:col10/1733620584514/Put/seqid=0 2024-12-08T01:16:24,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/932415c09cb54ffd89ef60d8b46f3cbc is 50, key is test_row_0/A:col10/1733620584752/Put/seqid=0 2024-12-08T01:16:24,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742298_1474 (size=12915) 2024-12-08T01:16:24,782 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/7ab1f179670b4c1fb3ced6ae19ed9968 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7ab1f179670b4c1fb3ced6ae19ed9968 2024-12-08T01:16:24,789 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into 7ab1f179670b4c1fb3ced6ae19ed9968(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:24,790 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:24,790 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=13, startTime=1733620584624; duration=0sec 2024-12-08T01:16:24,790 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:24,790 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742299_1475 (size=14741) 2024-12-08T01:16:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T01:16:24,832 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:24,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-08T01:16:24,835 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:24,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:24,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:24,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620584833, max timestamp=1733620584874Num uniques 859; 2024-12-08T01:16:24,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:24,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620584833, max timestamp=1733620584874Num uniques 859; 2024-12-08T01:16:24,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:24,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620584833, max timestamp=1733620584874Num uniques 859; 2024-12-08T01:16:24,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:24,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:16:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620644969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620644974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:24,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:24,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620645093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620645094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,111 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/243ba2155a3a4ce6a6eb9c981ca89d22 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/243ba2155a3a4ce6a6eb9c981ca89d22 2024-12-08T01:16:25,119 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into 243ba2155a3a4ce6a6eb9c981ca89d22(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:25,119 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:25,119 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=13, startTime=1733620584624; duration=0sec 2024-12-08T01:16:25,119 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:25,119 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:25,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620645193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,197 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/932415c09cb54ffd89ef60d8b46f3cbc 2024-12-08T01:16:25,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/4b3a7d0517ec4ff3a1256e9e93a93d4e is 50, key is test_row_0/B:col10/1733620584752/Put/seqid=0 2024-12-08T01:16:25,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742300_1476 (size=12301) 2024-12-08T01:16:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620645242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620645243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,248 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,248 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620645302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620645312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:25,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620645612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620645621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:25,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/4b3a7d0517ec4ff3a1256e9e93a93d4e 2024-12-08T01:16:25,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/66ba845fb5314b9a8c345939de9a2137 is 50, key is test_row_0/C:col10/1733620584752/Put/seqid=0 2024-12-08T01:16:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742301_1477 (size=12301) 2024-12-08T01:16:25,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:25,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:25,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:26,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/66ba845fb5314b9a8c345939de9a2137 2024-12-08T01:16:26,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/932415c09cb54ffd89ef60d8b46f3cbc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/932415c09cb54ffd89ef60d8b46f3cbc 2024-12-08T01:16:26,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/932415c09cb54ffd89ef60d8b46f3cbc, entries=200, sequenceid=290, filesize=14.4 K 2024-12-08T01:16:26,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/4b3a7d0517ec4ff3a1256e9e93a93d4e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4b3a7d0517ec4ff3a1256e9e93a93d4e 2024-12-08T01:16:26,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4b3a7d0517ec4ff3a1256e9e93a93d4e, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T01:16:26,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/66ba845fb5314b9a8c345939de9a2137 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/66ba845fb5314b9a8c345939de9a2137 2024-12-08T01:16:26,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/66ba845fb5314b9a8c345939de9a2137, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T01:16:26,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 192cb468141f6e111e56d290a9940203 in 1304ms, sequenceid=290, compaction requested=false 2024-12-08T01:16:26,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:26,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:26,059 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:26,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/51d42ad7aea344288ccdb7714a6e9454 is 50, key is test_row_0/A:col10/1733620584959/Put/seqid=0 2024-12-08T01:16:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742302_1478 (size=12301) 2024-12-08T01:16:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:26,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620646159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620646160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620646272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620646282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,468 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/51d42ad7aea344288ccdb7714a6e9454 2024-12-08T01:16:26,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/60e0112688224535a012e64bbdf27583 is 50, key is test_row_0/B:col10/1733620584959/Put/seqid=0 2024-12-08T01:16:26,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742303_1479 (size=12301) 2024-12-08T01:16:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620646492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620646502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620646802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620646811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:26,878 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/60e0112688224535a012e64bbdf27583 2024-12-08T01:16:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/417488eea781433bbf9d00576cd17d1b is 50, key is test_row_0/C:col10/1733620584959/Put/seqid=0 2024-12-08T01:16:26,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742304_1480 (size=12301) 2024-12-08T01:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620647222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,230 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620647252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,257 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620647262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,270 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,286 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/417488eea781433bbf9d00576cd17d1b 2024-12-08T01:16:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/51d42ad7aea344288ccdb7714a6e9454 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/51d42ad7aea344288ccdb7714a6e9454 2024-12-08T01:16:27,292 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/51d42ad7aea344288ccdb7714a6e9454, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T01:16:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/60e0112688224535a012e64bbdf27583 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/60e0112688224535a012e64bbdf27583 2024-12-08T01:16:27,296 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/60e0112688224535a012e64bbdf27583, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T01:16:27,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/417488eea781433bbf9d00576cd17d1b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/417488eea781433bbf9d00576cd17d1b 2024-12-08T01:16:27,299 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/417488eea781433bbf9d00576cd17d1b, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T01:16:27,300 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 192cb468141f6e111e56d290a9940203 in 1241ms, sequenceid=329, compaction requested=true 2024-12-08T01:16:27,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:27,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:27,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-08T01:16:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-08T01:16:27,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-08T01:16:27,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4640 sec 2024-12-08T01:16:27,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.4700 sec 2024-12-08T01:16:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:27,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:16:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:27,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/47c40f335717422da241f7cbd3718d5c is 50, key is test_row_0/A:col10/1733620586149/Put/seqid=0 2024-12-08T01:16:27,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742305_1481 (size=14741) 2024-12-08T01:16:27,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/47c40f335717422da241f7cbd3718d5c 2024-12-08T01:16:27,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/94f9686b7a23452bad7af45f53818110 is 50, key is test_row_0/B:col10/1733620586149/Put/seqid=0 2024-12-08T01:16:27,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742306_1482 (size=12301) 2024-12-08T01:16:27,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620587383, max timestamp=1733620587407Num uniques 859; 2024-12-08T01:16:27,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:27,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620587383, max timestamp=1733620587407Num uniques 859; 2024-12-08T01:16:27,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620587383, max timestamp=1733620587407Num uniques 859; 2024-12-08T01:16:27,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:27,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620647460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620647461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620647571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620647572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/94f9686b7a23452bad7af45f53818110 2024-12-08T01:16:27,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5cb8833a62eb4eb395e73af426839ccb is 50, key is test_row_0/C:col10/1733620586149/Put/seqid=0 2024-12-08T01:16:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742307_1483 (size=12301) 2024-12-08T01:16:27,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620647782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620647782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5cb8833a62eb4eb395e73af426839ccb 2024-12-08T01:16:28,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/47c40f335717422da241f7cbd3718d5c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/47c40f335717422da241f7cbd3718d5c 2024-12-08T01:16:28,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/47c40f335717422da241f7cbd3718d5c, entries=200, sequenceid=340, filesize=14.4 K 2024-12-08T01:16:28,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/94f9686b7a23452bad7af45f53818110 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/94f9686b7a23452bad7af45f53818110 2024-12-08T01:16:28,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/94f9686b7a23452bad7af45f53818110, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T01:16:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5cb8833a62eb4eb395e73af426839ccb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5cb8833a62eb4eb395e73af426839ccb 2024-12-08T01:16:28,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5cb8833a62eb4eb395e73af426839ccb, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T01:16:28,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 192cb468141f6e111e56d290a9940203 in 839ms, sequenceid=340, compaction requested=true 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:28,154 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:28,154 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:28,154 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:28,155 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:28,155 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/243ba2155a3a4ce6a6eb9c981ca89d22, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/932415c09cb54ffd89ef60d8b46f3cbc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/51d42ad7aea344288ccdb7714a6e9454, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/47c40f335717422da241f7cbd3718d5c] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=53.4 K 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:28,155 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243ba2155a3a4ce6a6eb9c981ca89d22, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620584164 2024-12-08T01:16:28,155 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/149fb160359c45539682913aeff476a3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4b3a7d0517ec4ff3a1256e9e93a93d4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/60e0112688224535a012e64bbdf27583, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/94f9686b7a23452bad7af45f53818110] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=48.7 K 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 932415c09cb54ffd89ef60d8b46f3cbc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733620584733 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 149fb160359c45539682913aeff476a3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620584164 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d42ad7aea344288ccdb7714a6e9454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733620584959 2024-12-08T01:16:28,155 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3a7d0517ec4ff3a1256e9e93a93d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733620584749 2024-12-08T01:16:28,156 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47c40f335717422da241f7cbd3718d5c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620586140 2024-12-08T01:16:28,156 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e0112688224535a012e64bbdf27583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733620584959 2024-12-08T01:16:28,156 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f9686b7a23452bad7af45f53818110, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620586149 2024-12-08T01:16:28,163 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:28,164 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/d61cc11497374914bbb5bada63635c5c is 50, key is test_row_0/B:col10/1733620586149/Put/seqid=0 2024-12-08T01:16:28,164 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:28,165 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/8d960edde62749d28daa7b63304f976e is 50, key is test_row_0/A:col10/1733620586149/Put/seqid=0 2024-12-08T01:16:28,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742309_1485 (size=13051) 2024-12-08T01:16:28,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742308_1484 (size=13051) 2024-12-08T01:16:28,572 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/8d960edde62749d28daa7b63304f976e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/8d960edde62749d28daa7b63304f976e 2024-12-08T01:16:28,574 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/d61cc11497374914bbb5bada63635c5c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/d61cc11497374914bbb5bada63635c5c 2024-12-08T01:16:28,576 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into 8d960edde62749d28daa7b63304f976e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:28,576 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:28,576 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=12, startTime=1733620588154; duration=0sec 2024-12-08T01:16:28,576 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:28,576 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:28,576 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:28,577 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into d61cc11497374914bbb5bada63635c5c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:28,577 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:28,577 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=12, startTime=1733620588154; duration=0sec 2024-12-08T01:16:28,577 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:28,577 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:28,577 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:28,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:28,578 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:28,578 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7ab1f179670b4c1fb3ced6ae19ed9968, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/66ba845fb5314b9a8c345939de9a2137, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/417488eea781433bbf9d00576cd17d1b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5cb8833a62eb4eb395e73af426839ccb] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=48.7 K 2024-12-08T01:16:28,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab1f179670b4c1fb3ced6ae19ed9968, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620584164 2024-12-08T01:16:28,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66ba845fb5314b9a8c345939de9a2137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733620584749 2024-12-08T01:16:28,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417488eea781433bbf9d00576cd17d1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733620584959 2024-12-08T01:16:28,578 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb8833a62eb4eb395e73af426839ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620586149 2024-12-08T01:16:28,584 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:28,585 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/244dc3f73480451ea877fc9c0e9a0bb0 is 50, key is test_row_0/C:col10/1733620586149/Put/seqid=0 2024-12-08T01:16:28,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742310_1486 (size=13051) 2024-12-08T01:16:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:28,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T01:16:28,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:28,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:28,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:28,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:28,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:28,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:28,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/da134d8da37046fe9e8beec97b5224c5 is 50, key is test_row_0/A:col10/1733620587460/Put/seqid=0 2024-12-08T01:16:28,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742311_1487 (size=14741) 2024-12-08T01:16:28,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620648637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620648643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620648752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620648762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620648962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620648972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:28,990 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/244dc3f73480451ea877fc9c0e9a0bb0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/244dc3f73480451ea877fc9c0e9a0bb0 2024-12-08T01:16:28,993 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into 244dc3f73480451ea877fc9c0e9a0bb0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:28,993 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:28,993 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=12, startTime=1733620588154; duration=0sec 2024-12-08T01:16:28,993 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:28,993 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T01:16:29,002 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:29,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-08T01:16:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T01:16:29,004 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:29,005 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:29,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:29,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/da134d8da37046fe9e8beec97b5224c5 2024-12-08T01:16:29,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/f16d4145956447908aeb2617e1975d3e is 50, key is test_row_0/B:col10/1733620587460/Put/seqid=0 2024-12-08T01:16:29,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742312_1488 (size=12301) 2024-12-08T01:16:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T01:16:29,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T01:16:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620649271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:29,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620649282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:29,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T01:16:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T01:16:29,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/f16d4145956447908aeb2617e1975d3e 2024-12-08T01:16:29,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5c0e72b4a22d48908b4bf87f753666d4 is 50, key is test_row_0/C:col10/1733620587460/Put/seqid=0 2024-12-08T01:16:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742313_1489 (size=12301) 2024-12-08T01:16:29,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T01:16:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T01:16:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T01:16:29,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T01:16:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:29,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620649782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:29,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620649792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:29,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5c0e72b4a22d48908b4bf87f753666d4 2024-12-08T01:16:29,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/da134d8da37046fe9e8beec97b5224c5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/da134d8da37046fe9e8beec97b5224c5 2024-12-08T01:16:29,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/da134d8da37046fe9e8beec97b5224c5, entries=200, sequenceid=378, filesize=14.4 K 2024-12-08T01:16:29,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/f16d4145956447908aeb2617e1975d3e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f16d4145956447908aeb2617e1975d3e 2024-12-08T01:16:29,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f16d4145956447908aeb2617e1975d3e, entries=150, sequenceid=378, filesize=12.0 K 2024-12-08T01:16:29,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/5c0e72b4a22d48908b4bf87f753666d4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5c0e72b4a22d48908b4bf87f753666d4 2024-12-08T01:16:29,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5c0e72b4a22d48908b4bf87f753666d4, entries=150, sequenceid=378, filesize=12.0 K 2024-12-08T01:16:29,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 192cb468141f6e111e56d290a9940203 in 1257ms, sequenceid=378, compaction requested=false 2024-12-08T01:16:29,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:29,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T01:16:29,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:29,922 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:16:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/2b81f5dc22f04f98b7e87967e7bd95ce is 50, key is test_row_0/A:col10/1733620588628/Put/seqid=0 2024-12-08T01:16:29,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742314_1490 (size=12301) 2024-12-08T01:16:29,971 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/2b81f5dc22f04f98b7e87967e7bd95ce 2024-12-08T01:16:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/4ae0d0cb9e664a979065248b86466247 is 50, key is test_row_0/B:col10/1733620588628/Put/seqid=0 2024-12-08T01:16:30,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742315_1491 (size=12301) 2024-12-08T01:16:30,010 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/4ae0d0cb9e664a979065248b86466247 2024-12-08T01:16:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/e1aa6ba450a04c048f81cd5642bd1483 is 50, key is test_row_0/C:col10/1733620588628/Put/seqid=0 2024-12-08T01:16:30,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742316_1492 (size=12301) 2024-12-08T01:16:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T01:16:30,479 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/e1aa6ba450a04c048f81cd5642bd1483 2024-12-08T01:16:30,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/2b81f5dc22f04f98b7e87967e7bd95ce as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2b81f5dc22f04f98b7e87967e7bd95ce 2024-12-08T01:16:30,488 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2b81f5dc22f04f98b7e87967e7bd95ce, entries=150, sequenceid=389, filesize=12.0 K 2024-12-08T01:16:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/4ae0d0cb9e664a979065248b86466247 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4ae0d0cb9e664a979065248b86466247 2024-12-08T01:16:30,494 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4ae0d0cb9e664a979065248b86466247, entries=150, sequenceid=389, filesize=12.0 K 2024-12-08T01:16:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/e1aa6ba450a04c048f81cd5642bd1483 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e1aa6ba450a04c048f81cd5642bd1483 2024-12-08T01:16:30,505 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e1aa6ba450a04c048f81cd5642bd1483, entries=150, sequenceid=389, filesize=12.0 K 2024-12-08T01:16:30,506 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 192cb468141f6e111e56d290a9940203 in 585ms, sequenceid=389, compaction requested=true 2024-12-08T01:16:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-08T01:16:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-08T01:16:30,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-08T01:16:30,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5030 sec 2024-12-08T01:16:30,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5180 sec 2024-12-08T01:16:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:30,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:16:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:30,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/2285bde9d4994ca7a498f4fc09a2d4e9 is 50, key is test_row_0/A:col10/1733620590824/Put/seqid=0 2024-12-08T01:16:30,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742317_1493 (size=14737) 2024-12-08T01:16:30,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/2285bde9d4994ca7a498f4fc09a2d4e9 2024-12-08T01:16:30,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6633a11a84f24251b0cdf6a580c7e77b is 50, key is test_row_0/B:col10/1733620590824/Put/seqid=0 2024-12-08T01:16:30,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742318_1494 (size=9857) 2024-12-08T01:16:30,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6633a11a84f24251b0cdf6a580c7e77b 2024-12-08T01:16:30,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/c1d0e562ba6942a4862e303c72c64edb is 50, key is test_row_0/C:col10/1733620590824/Put/seqid=0 2024-12-08T01:16:30,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620590902, max timestamp=1733620590954Num uniques 859; 2024-12-08T01:16:30,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:30,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620590902, max timestamp=1733620590954Num uniques 859; 2024-12-08T01:16:30,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:30,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620590902, max timestamp=1733620590954Num uniques 859; 2024-12-08T01:16:30,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:30,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742319_1495 (size=9857) 2024-12-08T01:16:30,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/c1d0e562ba6942a4862e303c72c64edb 2024-12-08T01:16:30,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/2285bde9d4994ca7a498f4fc09a2d4e9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2285bde9d4994ca7a498f4fc09a2d4e9 2024-12-08T01:16:30,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2285bde9d4994ca7a498f4fc09a2d4e9, entries=200, sequenceid=399, filesize=14.4 K 2024-12-08T01:16:30,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/6633a11a84f24251b0cdf6a580c7e77b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6633a11a84f24251b0cdf6a580c7e77b 2024-12-08T01:16:30,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6633a11a84f24251b0cdf6a580c7e77b, entries=100, sequenceid=399, filesize=9.6 K 2024-12-08T01:16:30,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/c1d0e562ba6942a4862e303c72c64edb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/c1d0e562ba6942a4862e303c72c64edb 2024-12-08T01:16:31,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/c1d0e562ba6942a4862e303c72c64edb, entries=100, sequenceid=399, filesize=9.6 K 2024-12-08T01:16:31,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=60.38 KB/61830 for 192cb468141f6e111e56d290a9940203 in 171ms, sequenceid=399, compaction requested=true 2024-12-08T01:16:31,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:31,002 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:31,002 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54830 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:31,003 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:31,003 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,003 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/8d960edde62749d28daa7b63304f976e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/da134d8da37046fe9e8beec97b5224c5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2b81f5dc22f04f98b7e87967e7bd95ce, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2285bde9d4994ca7a498f4fc09a2d4e9] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=53.5 K 2024-12-08T01:16:31,003 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d960edde62749d28daa7b63304f976e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620586149 2024-12-08T01:16:31,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:31,003 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting da134d8da37046fe9e8beec97b5224c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733620587445 2024-12-08T01:16:31,009 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b81f5dc22f04f98b7e87967e7bd95ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733620588628 2024-12-08T01:16:31,010 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2285bde9d4994ca7a498f4fc09a2d4e9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620590821 2024-12-08T01:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:31,015 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:31,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=67.09 KB heapSize=185.77 KB 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:31,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:31,027 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:31,027 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:31,027 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,027 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/d61cc11497374914bbb5bada63635c5c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f16d4145956447908aeb2617e1975d3e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4ae0d0cb9e664a979065248b86466247, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6633a11a84f24251b0cdf6a580c7e77b] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=46.4 K 2024-12-08T01:16:31,028 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d61cc11497374914bbb5bada63635c5c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620586149 2024-12-08T01:16:31,029 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f16d4145956447908aeb2617e1975d3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733620587445 2024-12-08T01:16:31,029 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae0d0cb9e664a979065248b86466247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733620588628 2024-12-08T01:16:31,030 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 6633a11a84f24251b0cdf6a580c7e77b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620590824 2024-12-08T01:16:31,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/e41bf1878de7434da805487011acf7e1 is 50, key is test_row_0/A:col10/1733620591010/Put/seqid=0 2024-12-08T01:16:31,055 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:31,055 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/65980033d8ef48c7a5bb195d4b556199 is 50, key is test_row_0/A:col10/1733620590824/Put/seqid=0 2024-12-08T01:16:31,075 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:31,076 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/51f25dc56f65429d98356567ac3ca779 is 50, key is test_row_0/B:col10/1733620590824/Put/seqid=0 2024-12-08T01:16:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742321_1497 (size=13187) 2024-12-08T01:16:31,106 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/65980033d8ef48c7a5bb195d4b556199 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/65980033d8ef48c7a5bb195d4b556199 2024-12-08T01:16:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742320_1496 (size=14741) 2024-12-08T01:16:31,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742322_1498 (size=13187) 2024-12-08T01:16:31,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/e41bf1878de7434da805487011acf7e1 2024-12-08T01:16:31,113 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into 65980033d8ef48c7a5bb195d4b556199(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:31,113 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:31,113 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=12, startTime=1733620591001; duration=0sec 2024-12-08T01:16:31,114 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:31,114 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:31,114 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:31,120 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/51f25dc56f65429d98356567ac3ca779 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/51f25dc56f65429d98356567ac3ca779 2024-12-08T01:16:31,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/af0f7fae704a4f82920d3650670ecb59 is 50, key is test_row_0/B:col10/1733620591010/Put/seqid=0 2024-12-08T01:16:31,128 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:31,128 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into 51f25dc56f65429d98356567ac3ca779(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:31,128 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:31,128 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:31,128 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=12, startTime=1733620591015; duration=0sec 2024-12-08T01:16:31,128 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,128 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:31,128 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/244dc3f73480451ea877fc9c0e9a0bb0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5c0e72b4a22d48908b4bf87f753666d4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e1aa6ba450a04c048f81cd5642bd1483, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/c1d0e562ba6942a4862e303c72c64edb] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=46.4 K 2024-12-08T01:16:31,128 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:31,129 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244dc3f73480451ea877fc9c0e9a0bb0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733620586149 2024-12-08T01:16:31,129 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0e72b4a22d48908b4bf87f753666d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733620587445 2024-12-08T01:16:31,129 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1aa6ba450a04c048f81cd5642bd1483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733620588628 2024-12-08T01:16:31,129 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d0e562ba6942a4862e303c72c64edb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620590824 2024-12-08T01:16:31,143 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:31,144 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f48a08ddc7464d8d88537065a06f4a84 is 50, key is test_row_0/C:col10/1733620590824/Put/seqid=0 2024-12-08T01:16:31,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620651139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T01:16:31,154 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:31,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-08T01:16:31,157 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T01:16:31,158 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:31,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:31,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742324_1500 (size=13187) 2024-12-08T01:16:31,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620651147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742323_1499 (size=12301) 2024-12-08T01:16:31,166 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f48a08ddc7464d8d88537065a06f4a84 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f48a08ddc7464d8d88537065a06f4a84 2024-12-08T01:16:31,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/af0f7fae704a4f82920d3650670ecb59 2024-12-08T01:16:31,173 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into f48a08ddc7464d8d88537065a06f4a84(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:31,173 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:31,173 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=12, startTime=1733620591016; duration=0sec 2024-12-08T01:16:31,173 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:31,173 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:31,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/9e76be1b6af24be39f69eae516ec0ad3 is 50, key is test_row_0/C:col10/1733620591010/Put/seqid=0 2024-12-08T01:16:31,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742325_1501 (size=12301) 2024-12-08T01:16:31,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1733620651242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,246 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T01:16:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620651262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620651271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60042 deadline: 1733620651282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,293 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T01:16:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60084 deadline: 1733620651302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,311 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 192cb468141f6e111e56d290a9940203 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T01:16:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T01:16:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620651473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620651491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/9e76be1b6af24be39f69eae516ec0ad3 2024-12-08T01:16:31,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/e41bf1878de7434da805487011acf7e1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e41bf1878de7434da805487011acf7e1 2024-12-08T01:16:31,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e41bf1878de7434da805487011acf7e1, entries=200, sequenceid=426, filesize=14.4 K 2024-12-08T01:16:31,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/af0f7fae704a4f82920d3650670ecb59 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af0f7fae704a4f82920d3650670ecb59 2024-12-08T01:16:31,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af0f7fae704a4f82920d3650670ecb59, entries=150, sequenceid=426, filesize=12.0 K 2024-12-08T01:16:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/9e76be1b6af24be39f69eae516ec0ad3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9e76be1b6af24be39f69eae516ec0ad3 2024-12-08T01:16:31,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9e76be1b6af24be39f69eae516ec0ad3, entries=150, sequenceid=426, filesize=12.0 K 2024-12-08T01:16:31,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for 192cb468141f6e111e56d290a9940203 in 582ms, sequenceid=426, compaction requested=false 2024-12-08T01:16:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:31,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T01:16:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,618 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:16:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:31,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/7cca3eef40844ca296722735b24e28f8 is 50, key is test_row_0/A:col10/1733620591146/Put/seqid=0 2024-12-08T01:16:31,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742326_1502 (size=12301) 2024-12-08T01:16:31,636 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/7cca3eef40844ca296722735b24e28f8 2024-12-08T01:16:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/f27418835ac24f58b353857d1aa6feb4 is 50, key is test_row_0/B:col10/1733620591146/Put/seqid=0 2024-12-08T01:16:31,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742327_1503 (size=12301) 2024-12-08T01:16:31,650 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/f27418835ac24f58b353857d1aa6feb4 2024-12-08T01:16:31,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/4f0e382b48634feeb6797c449ce14e5b is 50, key is test_row_0/C:col10/1733620591146/Put/seqid=0 2024-12-08T01:16:31,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742328_1504 (size=12301) 2024-12-08T01:16:31,671 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/4f0e382b48634feeb6797c449ce14e5b 2024-12-08T01:16:31,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/7cca3eef40844ca296722735b24e28f8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7cca3eef40844ca296722735b24e28f8 2024-12-08T01:16:31,684 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7cca3eef40844ca296722735b24e28f8, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T01:16:31,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/f27418835ac24f58b353857d1aa6feb4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f27418835ac24f58b353857d1aa6feb4 2024-12-08T01:16:31,689 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f27418835ac24f58b353857d1aa6feb4, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T01:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/4f0e382b48634feeb6797c449ce14e5b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/4f0e382b48634feeb6797c449ce14e5b 2024-12-08T01:16:31,694 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/4f0e382b48634feeb6797c449ce14e5b, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T01:16:31,695 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 192cb468141f6e111e56d290a9940203 in 78ms, sequenceid=449, compaction requested=true 2024-12-08T01:16:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-08T01:16:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-08T01:16:31,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-08T01:16:31,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 539 msec 2024-12-08T01:16:31,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 546 msec 2024-12-08T01:16:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T01:16:31,782 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:31,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-08T01:16:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:31,785 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:31,786 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:31,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:31,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T01:16:31,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:31,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:31,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:31,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:31,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:31,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:31,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/0eb07bb40f4840aabc2286858ff3f3e3 is 50, key is test_row_0/A:col10/1733620591798/Put/seqid=0 2024-12-08T01:16:31,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742329_1505 (size=14741) 2024-12-08T01:16:31,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620591865, max timestamp=1733620591879Num uniques 859; 2024-12-08T01:16:31,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620591865, max timestamp=1733620591879Num uniques 859; 2024-12-08T01:16:31,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:31,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:31,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733620591865, max timestamp=1733620591879Num uniques 859; 2024-12-08T01:16:31,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T01:16:31,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:31,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T01:16:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:31,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620651949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620651959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620652072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620652072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T01:16:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:32,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/0eb07bb40f4840aabc2286858ff3f3e3 2024-12-08T01:16:32,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c1d68912e219452093992c600f88c29c is 50, key is test_row_0/B:col10/1733620591798/Put/seqid=0 2024-12-08T01:16:32,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742330_1506 (size=12301) 2024-12-08T01:16:32,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T01:16:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620652282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:32,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620652282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T01:16:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:32,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T01:16:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:32,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60028 deadline: 1733620652591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:32,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60070 deadline: 1733620652592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 2024-12-08T01:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 , the old value is region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., hostname=96982154c400,44843,1733620436630, seqNum=2 because the exception is null or not the one we care about 2024-12-08T01:16:32,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c1d68912e219452093992c600f88c29c 2024-12-08T01:16:32,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/446e7183ae894e03a63cc926a8e798bc is 50, key is test_row_0/C:col10/1733620591798/Put/seqid=0 2024-12-08T01:16:32,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742331_1507 (size=12301) 2024-12-08T01:16:32,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/446e7183ae894e03a63cc926a8e798bc 2024-12-08T01:16:32,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/0eb07bb40f4840aabc2286858ff3f3e3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0eb07bb40f4840aabc2286858ff3f3e3 2024-12-08T01:16:32,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0eb07bb40f4840aabc2286858ff3f3e3, entries=200, sequenceid=459, filesize=14.4 K 2024-12-08T01:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/c1d68912e219452093992c600f88c29c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c1d68912e219452093992c600f88c29c 2024-12-08T01:16:32,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c1d68912e219452093992c600f88c29c, entries=150, sequenceid=459, filesize=12.0 K 2024-12-08T01:16:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/446e7183ae894e03a63cc926a8e798bc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/446e7183ae894e03a63cc926a8e798bc 2024-12-08T01:16:32,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/446e7183ae894e03a63cc926a8e798bc, entries=150, sequenceid=459, filesize=12.0 K 2024-12-08T01:16:32,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 192cb468141f6e111e56d290a9940203 in 881ms, sequenceid=459, compaction requested=true 2024-12-08T01:16:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:32,685 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:32,685 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:32,686 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:32,686 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/B is initiating minor compaction (all files) 2024-12-08T01:16:32,686 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/B in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,686 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/51f25dc56f65429d98356567ac3ca779, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af0f7fae704a4f82920d3650670ecb59, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f27418835ac24f58b353857d1aa6feb4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c1d68912e219452093992c600f88c29c] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=48.9 K 2024-12-08T01:16:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 192cb468141f6e111e56d290a9940203:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:32,687 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 51f25dc56f65429d98356567ac3ca779, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620588636 2024-12-08T01:16:32,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:32,687 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting af0f7fae704a4f82920d3650670ecb59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733620590953 2024-12-08T01:16:32,687 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:32,687 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/A is initiating minor compaction (all files) 2024-12-08T01:16:32,687 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/A in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,687 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/65980033d8ef48c7a5bb195d4b556199, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e41bf1878de7434da805487011acf7e1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7cca3eef40844ca296722735b24e28f8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0eb07bb40f4840aabc2286858ff3f3e3] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=53.7 K 2024-12-08T01:16:32,687 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65980033d8ef48c7a5bb195d4b556199, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620588636 2024-12-08T01:16:32,688 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f27418835ac24f58b353857d1aa6feb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733620591135 2024-12-08T01:16:32,688 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41bf1878de7434da805487011acf7e1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733620590953 2024-12-08T01:16:32,688 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cca3eef40844ca296722735b24e28f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733620591135 2024-12-08T01:16:32,688 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d68912e219452093992c600f88c29c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733620591792 2024-12-08T01:16:32,688 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb07bb40f4840aabc2286858ff3f3e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733620591782 2024-12-08T01:16:32,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,705 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:32,709 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:32,709 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/0d368e9ab7234c9288fae87b90d5ebe8 is 50, key is test_row_0/A:col10/1733620591798/Put/seqid=0 2024-12-08T01:16:32,711 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:32,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/87e61e51211a4bd5b06dd803c116819f is 50, key is test_row_0/A:col10/1733620591893/Put/seqid=0 2024-12-08T01:16:32,716 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/e273a0a211e54c20a932e839fb1cc6dc is 50, key is test_row_0/B:col10/1733620591798/Put/seqid=0 2024-12-08T01:16:32,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742333_1509 (size=12301) 2024-12-08T01:16:32,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742334_1510 (size=13323) 2024-12-08T01:16:32,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742332_1508 (size=13323) 2024-12-08T01:16:32,737 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/0d368e9ab7234c9288fae87b90d5ebe8 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0d368e9ab7234c9288fae87b90d5ebe8 2024-12-08T01:16:32,747 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/A of 192cb468141f6e111e56d290a9940203 into 0d368e9ab7234c9288fae87b90d5ebe8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:32,747 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:32,747 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/A, priority=12, startTime=1733620592685; duration=0sec 2024-12-08T01:16:32,747 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:32,747 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:A 2024-12-08T01:16:32,748 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:32,748 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-12-08T01:16:32,748 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:32,748 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,748 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:32,750 INFO [Thread-1906 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1906. 2024-12-08T01:16:32,750 DEBUG [Thread-1906 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:32,750 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,750 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:32,751 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:32,751 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 192cb468141f6e111e56d290a9940203/C is initiating minor compaction (all files) 2024-12-08T01:16:32,751 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 192cb468141f6e111e56d290a9940203/C in TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:32,751 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f48a08ddc7464d8d88537065a06f4a84, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9e76be1b6af24be39f69eae516ec0ad3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/4f0e382b48634feeb6797c449ce14e5b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/446e7183ae894e03a63cc926a8e798bc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp, totalSize=48.9 K 2024-12-08T01:16:32,752 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting f48a08ddc7464d8d88537065a06f4a84, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733620588636 2024-12-08T01:16:32,752 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e76be1b6af24be39f69eae516ec0ad3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733620590953 2024-12-08T01:16:32,752 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f0e382b48634feeb6797c449ce14e5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733620591135 2024-12-08T01:16:32,752 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 446e7183ae894e03a63cc926a8e798bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733620591792 2024-12-08T01:16:32,757 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-12-08T01:16:32,757 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:32,757 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,757 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:32,757 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-12-08T01:16:32,757 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:32,757 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,757 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:32,758 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-12-08T01:16:32,759 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:32,759 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,759 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:32,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:32,760 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 192cb468141f6e111e56d290a9940203#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:32,760 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/873eb0539e364c0f844d2995635853b3 is 50, key is test_row_0/C:col10/1733620591798/Put/seqid=0 2024-12-08T01:16:32,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742335_1511 (size=13323) 2024-12-08T01:16:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:33,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. as already flushing 2024-12-08T01:16:33,114 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-12-08T01:16:33,114 INFO [Thread-1895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1895. 2024-12-08T01:16:33,114 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:33,114 DEBUG [Thread-1895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:33,114 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:33,114 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:33,114 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:33,114 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:33,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:33,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:33,129 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/87e61e51211a4bd5b06dd803c116819f 2024-12-08T01:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/9564cec39f354b96bcfbcbf455cb1a8d is 50, key is test_row_0/B:col10/1733620591893/Put/seqid=0 2024-12-08T01:16:33,136 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/e273a0a211e54c20a932e839fb1cc6dc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e273a0a211e54c20a932e839fb1cc6dc 2024-12-08T01:16:33,139 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/B of 192cb468141f6e111e56d290a9940203 into e273a0a211e54c20a932e839fb1cc6dc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:33,139 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:33,139 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/B, priority=12, startTime=1733620592685; duration=0sec 2024-12-08T01:16:33,139 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:33,139 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:B 2024-12-08T01:16:33,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742336_1512 (size=12301) 2024-12-08T01:16:33,171 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/873eb0539e364c0f844d2995635853b3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/873eb0539e364c0f844d2995635853b3 2024-12-08T01:16:33,177 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 192cb468141f6e111e56d290a9940203/C of 192cb468141f6e111e56d290a9940203 into 873eb0539e364c0f844d2995635853b3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:33,177 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:33,177 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., storeName=192cb468141f6e111e56d290a9940203/C, priority=12, startTime=1733620592686; duration=0sec 2024-12-08T01:16:33,177 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:33,178 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 192cb468141f6e111e56d290a9940203:C 2024-12-08T01:16:33,542 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/9564cec39f354b96bcfbcbf455cb1a8d 2024-12-08T01:16:33,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/909b0d292cd34d97aa3567a28624268f is 50, key is test_row_0/C:col10/1733620591893/Put/seqid=0 2024-12-08T01:16:33,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742337_1513 (size=12301) 2024-12-08T01:16:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:33,952 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/909b0d292cd34d97aa3567a28624268f 2024-12-08T01:16:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/87e61e51211a4bd5b06dd803c116819f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/87e61e51211a4bd5b06dd803c116819f 2024-12-08T01:16:33,958 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/87e61e51211a4bd5b06dd803c116819f, entries=150, sequenceid=495, filesize=12.0 K 2024-12-08T01:16:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/9564cec39f354b96bcfbcbf455cb1a8d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9564cec39f354b96bcfbcbf455cb1a8d 2024-12-08T01:16:33,961 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9564cec39f354b96bcfbcbf455cb1a8d, entries=150, sequenceid=495, filesize=12.0 K 2024-12-08T01:16:33,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/909b0d292cd34d97aa3567a28624268f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/909b0d292cd34d97aa3567a28624268f 2024-12-08T01:16:33,965 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/909b0d292cd34d97aa3567a28624268f, entries=150, sequenceid=495, filesize=12.0 K 2024-12-08T01:16:33,966 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=13.42 KB/13740 for 192cb468141f6e111e56d290a9940203 in 1261ms, sequenceid=495, compaction requested=false 2024-12-08T01:16:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 192cb468141f6e111e56d290a9940203: 2024-12-08T01:16:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-08T01:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-08T01:16:33,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-08T01:16:33,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-12-08T01:16:33,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.1870 sec 2024-12-08T01:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T01:16:35,953 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:41,297 INFO [Thread-1897 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1897. 2024-12-08T01:16:41,297 DEBUG [Thread-1897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:41,297 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,298 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:41,386 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-12-08T01:16:41,387 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:41,387 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,387 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:41,417 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-12-08T01:16:41,417 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:16:41,417 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,417 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1547 2024-12-08T01:16:41,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4637 rows 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1546 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4638 rows 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1569 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4697 rows 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1560 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4680 rows 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1549 2024-12-08T01:16:41,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4647 rows 2024-12-08T01:16:41,419 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:16:41,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:16:41,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:41,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:16:41,422 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T01:16:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T01:16:41,426 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620601425"}]},"ts":"1733620601425"} 2024-12-08T01:16:41,427 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T01:16:41,427 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T01:16:41,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:16:41,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, UNASSIGN}] 2024-12-08T01:16:41,430 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, UNASSIGN 2024-12-08T01:16:41,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=192cb468141f6e111e56d290a9940203, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:41,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, UNASSIGN because future has completed 2024-12-08T01:16:41,432 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:16:41,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630}] 2024-12-08T01:16:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T01:16:41,586 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:41,586 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:16:41,586 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 192cb468141f6e111e56d290a9940203, disabling compactions & flushes 2024-12-08T01:16:41,586 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:41,586 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:41,586 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. after waiting 0 ms 2024-12-08T01:16:41,586 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:41,587 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 192cb468141f6e111e56d290a9940203 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T01:16:41,587 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=A 2024-12-08T01:16:41,587 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:41,587 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=B 2024-12-08T01:16:41,587 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:41,587 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 192cb468141f6e111e56d290a9940203, store=C 2024-12-08T01:16:41,587 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:41,594 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/5f6bc029ff314601a212384c2763295e is 50, key is test_row_0/A:col10/1733620601413/Put/seqid=0 2024-12-08T01:16:41,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742338_1514 (size=12301) 2024-12-08T01:16:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T01:16:42,001 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/5f6bc029ff314601a212384c2763295e 2024-12-08T01:16:42,012 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/cbc69c4e4a4b487c871494341afd67cd is 50, key is test_row_0/B:col10/1733620601413/Put/seqid=0 2024-12-08T01:16:42,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742339_1515 (size=12301) 2024-12-08T01:16:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T01:16:42,417 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/cbc69c4e4a4b487c871494341afd67cd 2024-12-08T01:16:42,422 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f80dd5823b964546a4b3b35e7fe4551d is 50, key is test_row_0/C:col10/1733620601413/Put/seqid=0 2024-12-08T01:16:42,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742340_1516 (size=12301) 2024-12-08T01:16:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T01:16:42,826 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f80dd5823b964546a4b3b35e7fe4551d 2024-12-08T01:16:42,830 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/A/5f6bc029ff314601a212384c2763295e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5f6bc029ff314601a212384c2763295e 2024-12-08T01:16:42,832 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5f6bc029ff314601a212384c2763295e, entries=150, sequenceid=506, filesize=12.0 K 2024-12-08T01:16:42,833 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/B/cbc69c4e4a4b487c871494341afd67cd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/cbc69c4e4a4b487c871494341afd67cd 2024-12-08T01:16:42,836 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/cbc69c4e4a4b487c871494341afd67cd, entries=150, sequenceid=506, filesize=12.0 K 2024-12-08T01:16:42,837 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/.tmp/C/f80dd5823b964546a4b3b35e7fe4551d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f80dd5823b964546a4b3b35e7fe4551d 2024-12-08T01:16:42,840 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f80dd5823b964546a4b3b35e7fe4551d, entries=150, sequenceid=506, filesize=12.0 K 2024-12-08T01:16:42,841 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 192cb468141f6e111e56d290a9940203 in 1255ms, sequenceid=506, compaction requested=true 2024-12-08T01:16:42,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/f52e08b272764743ac7104f5928f238b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ee7e6917e8f14427b7ede446ff0c1361, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5894345109ba4933b268b781990f731b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/fd221243e24f45c2baab84c285941830, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/3b1a8395b7e2459ebe83575da89ff00f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/9deb7e8a48444b8abdefff2da0891fbc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e4ae514423ad4ca4ab3bb1f2345eb429, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7460d59bd81544249972bb2f7a5f7482, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/c350f11333ee4ebaa0907e421973c364, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/4944ef947f3c4d7f9af694f882c32261, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/31bebb9437364e2fbb5b60a6dea3e854, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/27e110df0a0c473db2e7eac566539814, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ed45390e4870499aa1a733f2e967f958, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/216c60830e38496f830c4ad52a5e993d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/93c2bd08d0be46b6a51747e187970cfb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/cb7fecb3bfff436ab6496a48fb4d5bb9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/dcd15a300e6c431d92e03995e63c1263, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/caaa9ba6f0414e5485a171afdc4e6453, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/243ba2155a3a4ce6a6eb9c981ca89d22, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/932415c09cb54ffd89ef60d8b46f3cbc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/51d42ad7aea344288ccdb7714a6e9454, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/47c40f335717422da241f7cbd3718d5c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/8d960edde62749d28daa7b63304f976e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/da134d8da37046fe9e8beec97b5224c5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2b81f5dc22f04f98b7e87967e7bd95ce, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2285bde9d4994ca7a498f4fc09a2d4e9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/65980033d8ef48c7a5bb195d4b556199, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e41bf1878de7434da805487011acf7e1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7cca3eef40844ca296722735b24e28f8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0eb07bb40f4840aabc2286858ff3f3e3] to archive 2024-12-08T01:16:42,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:16:42,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/f52e08b272764743ac7104f5928f238b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/f52e08b272764743ac7104f5928f238b 2024-12-08T01:16:42,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ee7e6917e8f14427b7ede446ff0c1361 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ee7e6917e8f14427b7ede446ff0c1361 2024-12-08T01:16:42,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5894345109ba4933b268b781990f731b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5894345109ba4933b268b781990f731b 2024-12-08T01:16:42,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/fd221243e24f45c2baab84c285941830 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/fd221243e24f45c2baab84c285941830 2024-12-08T01:16:42,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/3b1a8395b7e2459ebe83575da89ff00f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/3b1a8395b7e2459ebe83575da89ff00f 2024-12-08T01:16:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/9deb7e8a48444b8abdefff2da0891fbc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/9deb7e8a48444b8abdefff2da0891fbc 2024-12-08T01:16:42,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e4ae514423ad4ca4ab3bb1f2345eb429 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e4ae514423ad4ca4ab3bb1f2345eb429 2024-12-08T01:16:42,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7460d59bd81544249972bb2f7a5f7482 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7460d59bd81544249972bb2f7a5f7482 2024-12-08T01:16:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/c350f11333ee4ebaa0907e421973c364 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/c350f11333ee4ebaa0907e421973c364 2024-12-08T01:16:42,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/4944ef947f3c4d7f9af694f882c32261 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/4944ef947f3c4d7f9af694f882c32261 2024-12-08T01:16:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/31bebb9437364e2fbb5b60a6dea3e854 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/31bebb9437364e2fbb5b60a6dea3e854 2024-12-08T01:16:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/27e110df0a0c473db2e7eac566539814 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/27e110df0a0c473db2e7eac566539814 2024-12-08T01:16:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ed45390e4870499aa1a733f2e967f958 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/ed45390e4870499aa1a733f2e967f958 2024-12-08T01:16:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/216c60830e38496f830c4ad52a5e993d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/216c60830e38496f830c4ad52a5e993d 2024-12-08T01:16:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/93c2bd08d0be46b6a51747e187970cfb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/93c2bd08d0be46b6a51747e187970cfb 2024-12-08T01:16:42,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/cb7fecb3bfff436ab6496a48fb4d5bb9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/cb7fecb3bfff436ab6496a48fb4d5bb9 2024-12-08T01:16:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/dcd15a300e6c431d92e03995e63c1263 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/dcd15a300e6c431d92e03995e63c1263 2024-12-08T01:16:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/caaa9ba6f0414e5485a171afdc4e6453 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/caaa9ba6f0414e5485a171afdc4e6453 2024-12-08T01:16:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/243ba2155a3a4ce6a6eb9c981ca89d22 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/243ba2155a3a4ce6a6eb9c981ca89d22 2024-12-08T01:16:42,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/932415c09cb54ffd89ef60d8b46f3cbc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/932415c09cb54ffd89ef60d8b46f3cbc 2024-12-08T01:16:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/51d42ad7aea344288ccdb7714a6e9454 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/51d42ad7aea344288ccdb7714a6e9454 2024-12-08T01:16:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/47c40f335717422da241f7cbd3718d5c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/47c40f335717422da241f7cbd3718d5c 2024-12-08T01:16:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/8d960edde62749d28daa7b63304f976e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/8d960edde62749d28daa7b63304f976e 2024-12-08T01:16:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/da134d8da37046fe9e8beec97b5224c5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/da134d8da37046fe9e8beec97b5224c5 2024-12-08T01:16:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2b81f5dc22f04f98b7e87967e7bd95ce to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2b81f5dc22f04f98b7e87967e7bd95ce 2024-12-08T01:16:42,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2285bde9d4994ca7a498f4fc09a2d4e9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/2285bde9d4994ca7a498f4fc09a2d4e9 2024-12-08T01:16:42,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/65980033d8ef48c7a5bb195d4b556199 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/65980033d8ef48c7a5bb195d4b556199 2024-12-08T01:16:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e41bf1878de7434da805487011acf7e1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/e41bf1878de7434da805487011acf7e1 2024-12-08T01:16:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7cca3eef40844ca296722735b24e28f8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/7cca3eef40844ca296722735b24e28f8 2024-12-08T01:16:42,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0eb07bb40f4840aabc2286858ff3f3e3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0eb07bb40f4840aabc2286858ff3f3e3 2024-12-08T01:16:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c406484ab2db4c9e93837e3b7cb98609, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6cb62d19b0a144b1a0d5720272436457, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e46ab0b8eceb4a3fa8902e3602111ff3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/8e0e1a009e0e4103bab2729bbe47be6f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1282c0efc435449d82da822d45b7b329, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e5a9e41acca8425295f81c7fc6f34d0e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c69e29a06f2c4f148f8f7b13722ce6b9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af6a42844aeb42a4b9249b911e40212a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/fe9d2bf177b04254acfc380a1948f47c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6810fc8ed25441c7a526cc2048feb237, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6161365aed234e47bc15944afaf57c55, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1aa0c8265c5947cb849ab303e94b45ae, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/a4013488b4d94901b7c6f7f5fc54b9a2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6e229304d1a4499ebe45a8c7dc16a649, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9e4216822e6a4ffcb1031bfb8c8ccf6e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/78bb9ec6f3b6408289f2e7dc3a268478, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/316cd29a67dc4709b29d7700a416c50f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/149fb160359c45539682913aeff476a3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/7c5797eebbd2452eb1bacd58bd44999c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4b3a7d0517ec4ff3a1256e9e93a93d4e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/60e0112688224535a012e64bbdf27583, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/d61cc11497374914bbb5bada63635c5c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/94f9686b7a23452bad7af45f53818110, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f16d4145956447908aeb2617e1975d3e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4ae0d0cb9e664a979065248b86466247, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/51f25dc56f65429d98356567ac3ca779, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6633a11a84f24251b0cdf6a580c7e77b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af0f7fae704a4f82920d3650670ecb59, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f27418835ac24f58b353857d1aa6feb4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c1d68912e219452093992c600f88c29c] to archive 2024-12-08T01:16:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:16:42,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c406484ab2db4c9e93837e3b7cb98609 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c406484ab2db4c9e93837e3b7cb98609 2024-12-08T01:16:42,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6cb62d19b0a144b1a0d5720272436457 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6cb62d19b0a144b1a0d5720272436457 2024-12-08T01:16:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e46ab0b8eceb4a3fa8902e3602111ff3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e46ab0b8eceb4a3fa8902e3602111ff3 2024-12-08T01:16:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/8e0e1a009e0e4103bab2729bbe47be6f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/8e0e1a009e0e4103bab2729bbe47be6f 2024-12-08T01:16:42,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1282c0efc435449d82da822d45b7b329 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1282c0efc435449d82da822d45b7b329 2024-12-08T01:16:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e5a9e41acca8425295f81c7fc6f34d0e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e5a9e41acca8425295f81c7fc6f34d0e 2024-12-08T01:16:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c69e29a06f2c4f148f8f7b13722ce6b9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c69e29a06f2c4f148f8f7b13722ce6b9 2024-12-08T01:16:42,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af6a42844aeb42a4b9249b911e40212a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af6a42844aeb42a4b9249b911e40212a 2024-12-08T01:16:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/fe9d2bf177b04254acfc380a1948f47c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/fe9d2bf177b04254acfc380a1948f47c 2024-12-08T01:16:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6810fc8ed25441c7a526cc2048feb237 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6810fc8ed25441c7a526cc2048feb237 2024-12-08T01:16:42,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6161365aed234e47bc15944afaf57c55 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6161365aed234e47bc15944afaf57c55 2024-12-08T01:16:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1aa0c8265c5947cb849ab303e94b45ae to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/1aa0c8265c5947cb849ab303e94b45ae 2024-12-08T01:16:42,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/a4013488b4d94901b7c6f7f5fc54b9a2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/a4013488b4d94901b7c6f7f5fc54b9a2 2024-12-08T01:16:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6e229304d1a4499ebe45a8c7dc16a649 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6e229304d1a4499ebe45a8c7dc16a649 2024-12-08T01:16:42,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9e4216822e6a4ffcb1031bfb8c8ccf6e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9e4216822e6a4ffcb1031bfb8c8ccf6e 2024-12-08T01:16:42,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/78bb9ec6f3b6408289f2e7dc3a268478 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/78bb9ec6f3b6408289f2e7dc3a268478 2024-12-08T01:16:42,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/316cd29a67dc4709b29d7700a416c50f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/316cd29a67dc4709b29d7700a416c50f 2024-12-08T01:16:42,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/149fb160359c45539682913aeff476a3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/149fb160359c45539682913aeff476a3 2024-12-08T01:16:42,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/7c5797eebbd2452eb1bacd58bd44999c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/7c5797eebbd2452eb1bacd58bd44999c 2024-12-08T01:16:42,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4b3a7d0517ec4ff3a1256e9e93a93d4e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4b3a7d0517ec4ff3a1256e9e93a93d4e 2024-12-08T01:16:42,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/60e0112688224535a012e64bbdf27583 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/60e0112688224535a012e64bbdf27583 2024-12-08T01:16:42,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/d61cc11497374914bbb5bada63635c5c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/d61cc11497374914bbb5bada63635c5c 2024-12-08T01:16:42,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/94f9686b7a23452bad7af45f53818110 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/94f9686b7a23452bad7af45f53818110 2024-12-08T01:16:42,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f16d4145956447908aeb2617e1975d3e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f16d4145956447908aeb2617e1975d3e 2024-12-08T01:16:42,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4ae0d0cb9e664a979065248b86466247 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/4ae0d0cb9e664a979065248b86466247 2024-12-08T01:16:42,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/51f25dc56f65429d98356567ac3ca779 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/51f25dc56f65429d98356567ac3ca779 2024-12-08T01:16:42,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6633a11a84f24251b0cdf6a580c7e77b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/6633a11a84f24251b0cdf6a580c7e77b 2024-12-08T01:16:42,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af0f7fae704a4f82920d3650670ecb59 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/af0f7fae704a4f82920d3650670ecb59 2024-12-08T01:16:42,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f27418835ac24f58b353857d1aa6feb4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/f27418835ac24f58b353857d1aa6feb4 2024-12-08T01:16:42,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c1d68912e219452093992c600f88c29c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/c1d68912e219452093992c600f88c29c 2024-12-08T01:16:42,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/3210f03324a0433993c93b643fce4202, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5ee0de54c07546739ce9edc3123c44d8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5b42c44dd25b4c81b0f0d6a6256e84b6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f02df9118b0744d4aafbea30ab32b3a6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/0ca77a5c426945268d0c3c2d00ac9048, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/b04bc96506294de78382b5d675028477, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1bbb1468fbfd458e8ff06fad523b8ef6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/87bad5c84925402999b0b078ee9ede60, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a8efdc8050284a6c9d7bf87ae282886a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/562c24e2670548659396c9855accff6b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a4958a21b74e49f39833ad4429713c71, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1393c0f222ab462cb05d6efe298c39c8, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9653d01a316a4648a88f851caf53d439, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/ee1417430475475aa48ad1e68998442e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/53d3d1316b294856a552a372c21ee4eb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/886cf571069f4f68ad2ac2de964bd44e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e5774f8ca15c44e6adede851ee290055, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7ab1f179670b4c1fb3ced6ae19ed9968, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7bd1f8bd9334441589c95ba4104b200a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/66ba845fb5314b9a8c345939de9a2137, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/417488eea781433bbf9d00576cd17d1b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/244dc3f73480451ea877fc9c0e9a0bb0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5cb8833a62eb4eb395e73af426839ccb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5c0e72b4a22d48908b4bf87f753666d4, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e1aa6ba450a04c048f81cd5642bd1483, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f48a08ddc7464d8d88537065a06f4a84, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/c1d0e562ba6942a4862e303c72c64edb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9e76be1b6af24be39f69eae516ec0ad3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/4f0e382b48634feeb6797c449ce14e5b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/446e7183ae894e03a63cc926a8e798bc] to archive 2024-12-08T01:16:42,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:16:42,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/3210f03324a0433993c93b643fce4202 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/3210f03324a0433993c93b643fce4202 2024-12-08T01:16:42,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5ee0de54c07546739ce9edc3123c44d8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5ee0de54c07546739ce9edc3123c44d8 2024-12-08T01:16:42,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5b42c44dd25b4c81b0f0d6a6256e84b6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5b42c44dd25b4c81b0f0d6a6256e84b6 2024-12-08T01:16:42,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f02df9118b0744d4aafbea30ab32b3a6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f02df9118b0744d4aafbea30ab32b3a6 2024-12-08T01:16:42,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/0ca77a5c426945268d0c3c2d00ac9048 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/0ca77a5c426945268d0c3c2d00ac9048 2024-12-08T01:16:42,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/b04bc96506294de78382b5d675028477 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/b04bc96506294de78382b5d675028477 2024-12-08T01:16:42,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1bbb1468fbfd458e8ff06fad523b8ef6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1bbb1468fbfd458e8ff06fad523b8ef6 2024-12-08T01:16:42,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/87bad5c84925402999b0b078ee9ede60 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/87bad5c84925402999b0b078ee9ede60 2024-12-08T01:16:42,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a8efdc8050284a6c9d7bf87ae282886a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a8efdc8050284a6c9d7bf87ae282886a 2024-12-08T01:16:42,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/562c24e2670548659396c9855accff6b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/562c24e2670548659396c9855accff6b 2024-12-08T01:16:42,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a4958a21b74e49f39833ad4429713c71 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/a4958a21b74e49f39833ad4429713c71 2024-12-08T01:16:42,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1393c0f222ab462cb05d6efe298c39c8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/1393c0f222ab462cb05d6efe298c39c8 2024-12-08T01:16:42,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9653d01a316a4648a88f851caf53d439 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9653d01a316a4648a88f851caf53d439 2024-12-08T01:16:42,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/ee1417430475475aa48ad1e68998442e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/ee1417430475475aa48ad1e68998442e 2024-12-08T01:16:42,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/53d3d1316b294856a552a372c21ee4eb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/53d3d1316b294856a552a372c21ee4eb 2024-12-08T01:16:42,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/886cf571069f4f68ad2ac2de964bd44e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/886cf571069f4f68ad2ac2de964bd44e 2024-12-08T01:16:42,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e5774f8ca15c44e6adede851ee290055 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e5774f8ca15c44e6adede851ee290055 2024-12-08T01:16:42,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7ab1f179670b4c1fb3ced6ae19ed9968 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7ab1f179670b4c1fb3ced6ae19ed9968 2024-12-08T01:16:42,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7bd1f8bd9334441589c95ba4104b200a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/7bd1f8bd9334441589c95ba4104b200a 2024-12-08T01:16:42,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/66ba845fb5314b9a8c345939de9a2137 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/66ba845fb5314b9a8c345939de9a2137 2024-12-08T01:16:42,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/417488eea781433bbf9d00576cd17d1b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/417488eea781433bbf9d00576cd17d1b 2024-12-08T01:16:42,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/244dc3f73480451ea877fc9c0e9a0bb0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/244dc3f73480451ea877fc9c0e9a0bb0 2024-12-08T01:16:42,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5cb8833a62eb4eb395e73af426839ccb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5cb8833a62eb4eb395e73af426839ccb 2024-12-08T01:16:42,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5c0e72b4a22d48908b4bf87f753666d4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/5c0e72b4a22d48908b4bf87f753666d4 2024-12-08T01:16:42,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e1aa6ba450a04c048f81cd5642bd1483 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/e1aa6ba450a04c048f81cd5642bd1483 2024-12-08T01:16:42,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f48a08ddc7464d8d88537065a06f4a84 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f48a08ddc7464d8d88537065a06f4a84 2024-12-08T01:16:42,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/c1d0e562ba6942a4862e303c72c64edb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/c1d0e562ba6942a4862e303c72c64edb 2024-12-08T01:16:42,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9e76be1b6af24be39f69eae516ec0ad3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/9e76be1b6af24be39f69eae516ec0ad3 2024-12-08T01:16:42,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/4f0e382b48634feeb6797c449ce14e5b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/4f0e382b48634feeb6797c449ce14e5b 2024-12-08T01:16:42,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/446e7183ae894e03a63cc926a8e798bc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/446e7183ae894e03a63cc926a8e798bc 2024-12-08T01:16:42,930 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/recovered.edits/509.seqid, newMaxSeqId=509, maxSeqId=1 2024-12-08T01:16:42,931 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. 2024-12-08T01:16:42,931 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 192cb468141f6e111e56d290a9940203: Waiting for close lock at 1733620601586Running coprocessor pre-close hooks at 1733620601586Disabling compacts and flushes for region at 1733620601586Disabling writes for close at 1733620601586Obtaining lock to block concurrent updates at 1733620601587 (+1 ms)Preparing flush snapshotting stores in 192cb468141f6e111e56d290a9940203 at 1733620601587Finished memstore snapshotting TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733620601588 (+1 ms)Flushing stores of TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203. at 1733620601589 (+1 ms)Flushing 192cb468141f6e111e56d290a9940203/A: creating writer at 1733620601589Flushing 192cb468141f6e111e56d290a9940203/A: appending metadata at 1733620601594 (+5 ms)Flushing 192cb468141f6e111e56d290a9940203/A: closing flushed file at 1733620601594Flushing 192cb468141f6e111e56d290a9940203/B: creating writer at 1733620602009 (+415 ms)Flushing 192cb468141f6e111e56d290a9940203/B: appending metadata at 1733620602012 (+3 ms)Flushing 192cb468141f6e111e56d290a9940203/B: closing flushed file at 1733620602012Flushing 192cb468141f6e111e56d290a9940203/C: creating writer at 1733620602420 (+408 ms)Flushing 192cb468141f6e111e56d290a9940203/C: appending metadata at 1733620602422 (+2 ms)Flushing 192cb468141f6e111e56d290a9940203/C: closing flushed file at 1733620602422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53e9fa3c: reopening flushed file at 1733620602829 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31b2f8a3: reopening flushed file at 1733620602832 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fae7e94: reopening flushed file at 1733620602836 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 192cb468141f6e111e56d290a9940203 in 1255ms, sequenceid=506, compaction requested=true at 1733620602841 (+5 ms)Writing region close event to WAL at 1733620602928 (+87 ms)Running coprocessor post-close hooks at 1733620602931 (+3 ms)Closed at 1733620602931 2024-12-08T01:16:42,933 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 192cb468141f6e111e56d290a9940203 2024-12-08T01:16:42,934 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=192cb468141f6e111e56d290a9940203, regionState=CLOSED 2024-12-08T01:16:42,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:16:42,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-08T01:16:42,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 192cb468141f6e111e56d290a9940203, server=96982154c400,44843,1733620436630 in 1.5040 sec 2024-12-08T01:16:42,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-08T01:16:42,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=192cb468141f6e111e56d290a9940203, UNASSIGN in 1.5090 sec 2024-12-08T01:16:42,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-08T01:16:42,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5130 sec 2024-12-08T01:16:42,944 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620602943"}]},"ts":"1733620602943"} 2024-12-08T01:16:42,945 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T01:16:42,945 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T01:16:42,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5240 sec 2024-12-08T01:16:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T01:16:43,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:43,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T01:16:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,574 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T01:16:43,575 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,577 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203 2024-12-08T01:16:43,580 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/recovered.edits] 2024-12-08T01:16:43,583 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0d368e9ab7234c9288fae87b90d5ebe8 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/0d368e9ab7234c9288fae87b90d5ebe8 2024-12-08T01:16:43,584 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5f6bc029ff314601a212384c2763295e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/5f6bc029ff314601a212384c2763295e 2024-12-08T01:16:43,585 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/87e61e51211a4bd5b06dd803c116819f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/A/87e61e51211a4bd5b06dd803c116819f 2024-12-08T01:16:43,587 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9564cec39f354b96bcfbcbf455cb1a8d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/9564cec39f354b96bcfbcbf455cb1a8d 2024-12-08T01:16:43,588 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/cbc69c4e4a4b487c871494341afd67cd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/cbc69c4e4a4b487c871494341afd67cd 2024-12-08T01:16:43,589 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e273a0a211e54c20a932e839fb1cc6dc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/B/e273a0a211e54c20a932e839fb1cc6dc 2024-12-08T01:16:43,591 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/873eb0539e364c0f844d2995635853b3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/873eb0539e364c0f844d2995635853b3 2024-12-08T01:16:43,592 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/909b0d292cd34d97aa3567a28624268f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/909b0d292cd34d97aa3567a28624268f 2024-12-08T01:16:43,593 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f80dd5823b964546a4b3b35e7fe4551d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/C/f80dd5823b964546a4b3b35e7fe4551d 2024-12-08T01:16:43,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/recovered.edits/509.seqid to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203/recovered.edits/509.seqid 2024-12-08T01:16:43,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/192cb468141f6e111e56d290a9940203 2024-12-08T01:16:43,597 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T01:16:43,598 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,599 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T01:16:43,601 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T01:16:43,601 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,602 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T01:16:43,602 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733620603602"}]},"ts":"9223372036854775807"} 2024-12-08T01:16:43,603 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T01:16:43,603 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 192cb468141f6e111e56d290a9940203, NAME => 'TestAcidGuarantees,,1733620570540.192cb468141f6e111e56d290a9940203.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T01:16:43,603 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T01:16:43,603 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733620603603"}]},"ts":"9223372036854775807"} 2024-12-08T01:16:43,604 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T01:16:43,605 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-08T01:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T01:16:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T01:16:43,683 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:43,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 252), OpenFileDescriptor=445 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=474 (was 452) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10176 (was 10408) 2024-12-08T01:16:43,708 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=10176 2024-12-08T01:16:43,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:16:43,710 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:16:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:43,711 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:16:43,712 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:43,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-08T01:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T01:16:43,712 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:16:43,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742341_1517 (size=1061) 2024-12-08T01:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T01:16:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T01:16:44,119 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720 2024-12-08T01:16:44,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742342_1518 (size=53) 2024-12-08T01:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T01:16:44,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:16:44,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 375eeb2eab877f849984ffb3528951c4, disabling compactions & flushes 2024-12-08T01:16:44,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. after waiting 0 ms 2024-12-08T01:16:44,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 375eeb2eab877f849984ffb3528951c4: Waiting for close lock at 1733620604524Disabling compacts and flushes for region at 1733620604524Disabling writes for close at 1733620604524Writing region close event to WAL at 1733620604524Closed at 1733620604524 2024-12-08T01:16:44,525 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:16:44,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733620604525"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620604525"}]},"ts":"1733620604525"} 2024-12-08T01:16:44,527 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:16:44,528 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:16:44,528 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620604528"}]},"ts":"1733620604528"} 2024-12-08T01:16:44,529 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T01:16:44,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, ASSIGN}] 2024-12-08T01:16:44,530 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, ASSIGN 2024-12-08T01:16:44,531 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, ASSIGN; state=OFFLINE, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=false 2024-12-08T01:16:44,682 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:44,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, ASSIGN because future has completed 2024-12-08T01:16:44,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630}] 2024-12-08T01:16:44,840 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,840 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:16:44,841 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,841 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:16:44,841 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,841 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,843 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,844 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:44,844 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 375eeb2eab877f849984ffb3528951c4 columnFamilyName A 2024-12-08T01:16:44,844 DEBUG [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:44,845 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(327): Store=375eeb2eab877f849984ffb3528951c4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:44,845 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,846 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:44,847 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 375eeb2eab877f849984ffb3528951c4 columnFamilyName B 2024-12-08T01:16:44,847 DEBUG [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:44,847 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(327): Store=375eeb2eab877f849984ffb3528951c4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:44,847 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,848 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:44,849 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 375eeb2eab877f849984ffb3528951c4 columnFamilyName C 2024-12-08T01:16:44,849 DEBUG [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:44,849 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(327): Store=375eeb2eab877f849984ffb3528951c4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:44,850 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,850 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,851 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,851 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T01:16:44,852 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,853 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,853 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:16:44,854 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,856 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:16:44,856 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 375eeb2eab877f849984ffb3528951c4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64862503, jitterRate=-0.033473387360572815}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:16:44,856 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:44,857 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 375eeb2eab877f849984ffb3528951c4: Running coprocessor pre-open hook at 1733620604841Writing region info on filesystem at 1733620604841Initializing all the Stores at 1733620604842 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620604842Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620604842Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620604842Cleaning up temporary data from old regions at 1733620604853 (+11 ms)Running coprocessor post-open hooks at 1733620604856 (+3 ms)Region opened successfully at 1733620604857 (+1 ms) 2024-12-08T01:16:44,857 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., pid=146, masterSystemTime=1733620604837 2024-12-08T01:16:44,859 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,859 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:44,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:44,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:16:44,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-08T01:16:44,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 in 177 msec 2024-12-08T01:16:44,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-08T01:16:44,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, ASSIGN in 333 msec 2024-12-08T01:16:44,865 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:16:44,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620604865"}]},"ts":"1733620604865"} 2024-12-08T01:16:44,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T01:16:44,867 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:16:44,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-12-08T01:16:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T01:16:45,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:45,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:45,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:45,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:45,866 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22da96d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:45,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323d6ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:45,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:45,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:45,872 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,37013,1733620435769 2024-12-08T01:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@435c765d 2024-12-08T01:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:16:45,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:16:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T01:16:45,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:16:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T01:16:45,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742343_1519 (size=1097) 2024-12-08T01:16:46,289 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T01:16:46,289 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T01:16:46,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:16:46,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, REOPEN/MOVE}] 2024-12-08T01:16:46,300 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, REOPEN/MOVE 2024-12-08T01:16:46,301 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:46,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, REOPEN/MOVE because future has completed 2024-12-08T01:16:46,303 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:16:46,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630}] 2024-12-08T01:16:46,456 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,456 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:16:46,456 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 375eeb2eab877f849984ffb3528951c4, disabling compactions & flushes 2024-12-08T01:16:46,456 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,456 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,456 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. after waiting 0 ms 2024-12-08T01:16:46,456 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,464 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T01:16:46,465 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,465 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 375eeb2eab877f849984ffb3528951c4: Waiting for close lock at 1733620606456Running coprocessor pre-close hooks at 1733620606456Disabling compacts and flushes for region at 1733620606456Disabling writes for close at 1733620606456Writing region close event to WAL at 1733620606457 (+1 ms)Running coprocessor post-close hooks at 1733620606465 (+8 ms)Closed at 1733620606465 2024-12-08T01:16:46,466 WARN [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 375eeb2eab877f849984ffb3528951c4 to self. 2024-12-08T01:16:46,468 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=CLOSED 2024-12-08T01:16:46,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:16:46,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-08T01:16:46,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 in 168 msec 2024-12-08T01:16:46,474 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, REOPEN/MOVE; state=CLOSED, location=96982154c400,44843,1733620436630; forceNewPlan=false, retain=true 2024-12-08T01:16:46,625 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=OPENING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:46,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, REOPEN/MOVE because future has completed 2024-12-08T01:16:46,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630}] 2024-12-08T01:16:46,785 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,785 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:16:46,786 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,786 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:16:46,786 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,786 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,789 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,791 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:46,791 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 375eeb2eab877f849984ffb3528951c4 columnFamilyName A 2024-12-08T01:16:46,793 DEBUG [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:46,793 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(327): Store=375eeb2eab877f849984ffb3528951c4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:46,794 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,794 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:46,794 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 375eeb2eab877f849984ffb3528951c4 columnFamilyName B 2024-12-08T01:16:46,794 DEBUG [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:46,794 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(327): Store=375eeb2eab877f849984ffb3528951c4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:46,794 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,795 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T01:16:46,795 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 375eeb2eab877f849984ffb3528951c4 columnFamilyName C 2024-12-08T01:16:46,795 DEBUG [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:46,795 INFO [StoreOpener-375eeb2eab877f849984ffb3528951c4-1 {}] regionserver.HStore(327): Store=375eeb2eab877f849984ffb3528951c4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:16:46,795 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,795 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,796 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,797 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,798 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,798 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,798 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:16:46,799 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,800 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 375eeb2eab877f849984ffb3528951c4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68950188, jitterRate=0.027437865734100342}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:16:46,800 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,800 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 375eeb2eab877f849984ffb3528951c4: Running coprocessor pre-open hook at 1733620606787Writing region info on filesystem at 1733620606787Initializing all the Stores at 1733620606788 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620606788Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620606789 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620606789Cleaning up temporary data from old regions at 1733620606798 (+9 ms)Running coprocessor post-open hooks at 1733620606800 (+2 ms)Region opened successfully at 1733620606800 2024-12-08T01:16:46,801 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., pid=151, masterSystemTime=1733620606779 2024-12-08T01:16:46,802 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,802 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:46,802 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=OPEN, openSeqNum=5, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:16:46,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:16:46,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-08T01:16:46,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 in 177 msec 2024-12-08T01:16:46,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-08T01:16:46,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, REOPEN/MOVE in 508 msec 2024-12-08T01:16:46,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-08T01:16:46,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-12-08T01:16:46,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-12-08T01:16:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-08T01:16:46,813 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:46,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,814 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686d052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6f2f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,817 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c14d5e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf8564b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,821 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37bfa293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eeed4bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4427c945, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13080fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e68179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cfa8338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae12775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39385aa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1297acbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fbdc88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d1380a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@249d3961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,838 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e8f33a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,839 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2490f95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,37013,-1 for getting cluster id 2024-12-08T01:16:46,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:16:46,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee334ca5-882c-4f4a-9655-afec0198aa70' 2024-12-08T01:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee334ca5-882c-4f4a-9655-afec0198aa70" 2024-12-08T01:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbaf580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,37013,-1] 2024-12-08T01:16:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:16:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:16:46,842 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:16:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31bb308b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:16:46,843 DEBUG [Thread-2351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,843 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,843 DEBUG [Thread-2349 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,843 DEBUG [Thread-2338 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,844 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,844 DEBUG [Thread-2353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,844 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,844 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,844 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,845 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:46,845 DEBUG [Thread-2349 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,846 DEBUG [Thread-2349 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,44843,1733620436630, seqNum=-1] 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:16:46,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-08T01:16:46,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:46,848 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:46,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:16:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,849 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:46,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5] 2024-12-08T01:16:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:46,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T01:16:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:46,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620666867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620666869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620666870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe144d9539bd4f7f9d64764d34040131_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620606852/Put/seqid=0 2024-12-08T01:16:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620666870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620666871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742344_1520 (size=12154) 2024-12-08T01:16:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620666982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620666982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620666982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620666982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620666982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620667192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620667192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620667192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620667192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620667192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:47,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe144d9539bd4f7f9d64764d34040131_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe144d9539bd4f7f9d64764d34040131_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:47,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/7ea3ba98b9804c5cbab7886c81fc1441, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:47,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/7ea3ba98b9804c5cbab7886c81fc1441 is 175, key is test_row_0/A:col10/1733620606852/Put/seqid=0 2024-12-08T01:16:47,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742345_1521 (size=30955) 2024-12-08T01:16:47,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:47,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620667501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620667501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620667501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620667502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620667502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:47,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,689 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/7ea3ba98b9804c5cbab7886c81fc1441 2024-12-08T01:16:47,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b29b003cf4e04cb1ae4afd8dbd7df51b is 50, key is test_row_0/B:col10/1733620606852/Put/seqid=0 2024-12-08T01:16:47,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742346_1522 (size=12001) 2024-12-08T01:16:47,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b29b003cf4e04cb1ae4afd8dbd7df51b 2024-12-08T01:16:47,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/d3b82b087c49407e908ae2c36d641e8d is 50, key is test_row_0/C:col10/1733620606852/Put/seqid=0 2024-12-08T01:16:47,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742347_1523 (size=12001) 2024-12-08T01:16:47,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:47,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620668011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620668012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620668012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620668012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620668012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:48,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:48,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:48,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/d3b82b087c49407e908ae2c36d641e8d 2024-12-08T01:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/7ea3ba98b9804c5cbab7886c81fc1441 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441 2024-12-08T01:16:48,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441, entries=150, sequenceid=16, filesize=30.2 K 2024-12-08T01:16:48,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b29b003cf4e04cb1ae4afd8dbd7df51b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b29b003cf4e04cb1ae4afd8dbd7df51b 2024-12-08T01:16:48,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b29b003cf4e04cb1ae4afd8dbd7df51b, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T01:16:48,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/d3b82b087c49407e908ae2c36d641e8d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d3b82b087c49407e908ae2c36d641e8d 2024-12-08T01:16:48,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d3b82b087c49407e908ae2c36d641e8d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T01:16:48,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 375eeb2eab877f849984ffb3528951c4 in 1308ms, sequenceid=16, compaction requested=false 2024-12-08T01:16:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:48,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:48,223 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089956ea4ef3f84dec922a804887ef854f_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620606869/Put/seqid=0 2024-12-08T01:16:48,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742348_1524 (size=12154) 2024-12-08T01:16:48,450 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:16:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:48,633 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089956ea4ef3f84dec922a804887ef854f_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089956ea4ef3f84dec922a804887ef854f_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/a673a6eb9e9448c292475ce9e05770f1, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/a673a6eb9e9448c292475ce9e05770f1 is 175, key is test_row_0/A:col10/1733620606869/Put/seqid=0 2024-12-08T01:16:48,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742349_1525 (size=30955) 2024-12-08T01:16:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:49,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:49,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620669031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620669031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,034 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:49,035 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,037 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/a673a6eb9e9448c292475ce9e05770f1 2024-12-08T01:16:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620669033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620669034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620669034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/bfc3ddb88ed4406dbf2a3600f72044e0 is 50, key is test_row_0/B:col10/1733620606869/Put/seqid=0 2024-12-08T01:16:49,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742350_1526 (size=12001) 2024-12-08T01:16:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620669141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620669141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620669141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620669352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620669352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620669352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,453 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/bfc3ddb88ed4406dbf2a3600f72044e0 2024-12-08T01:16:49,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/1358d78667b1448b9d8b75dc661d4241 is 50, key is test_row_0/C:col10/1733620606869/Put/seqid=0 2024-12-08T01:16:49,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742351_1527 (size=12001) 2024-12-08T01:16:49,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620669662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620669662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620669662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:49,865 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/1358d78667b1448b9d8b75dc661d4241 2024-12-08T01:16:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/a673a6eb9e9448c292475ce9e05770f1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1 2024-12-08T01:16:49,872 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1, entries=150, sequenceid=36, filesize=30.2 K 2024-12-08T01:16:49,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/bfc3ddb88ed4406dbf2a3600f72044e0 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/bfc3ddb88ed4406dbf2a3600f72044e0 2024-12-08T01:16:49,876 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/bfc3ddb88ed4406dbf2a3600f72044e0, entries=150, sequenceid=36, filesize=11.7 K 2024-12-08T01:16:49,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/1358d78667b1448b9d8b75dc661d4241 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/1358d78667b1448b9d8b75dc661d4241 2024-12-08T01:16:49,881 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/1358d78667b1448b9d8b75dc661d4241, entries=150, sequenceid=36, filesize=11.7 K 2024-12-08T01:16:49,882 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 375eeb2eab877f849984ffb3528951c4 in 1659ms, sequenceid=36, compaction requested=false 2024-12-08T01:16:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-08T01:16:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-08T01:16:49,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-08T01:16:49,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0350 sec 2024-12-08T01:16:49,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0420 sec 2024-12-08T01:16:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:50,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T01:16:50,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:50,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:50,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:50,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:50,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:50,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:50,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082fcbebb938c9475081c71ec7a5e0b8a1_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:50,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742352_1528 (size=14594) 2024-12-08T01:16:50,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:50,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082fcbebb938c9475081c71ec7a5e0b8a1_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082fcbebb938c9475081c71ec7a5e0b8a1_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:50,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/8dcc1a1459824369acf5bb0658a5ada3, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:50,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/8dcc1a1459824369acf5bb0658a5ada3 is 175, key is test_row_0/A:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:50,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742353_1529 (size=39549) 2024-12-08T01:16:50,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620670197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620670198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620670198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620670312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620670312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620670312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620670522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620670522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620670522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/8dcc1a1459824369acf5bb0658a5ada3 2024-12-08T01:16:50,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/38700f0469d2479996af1cc7d1515815 is 50, key is test_row_0/B:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:50,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742354_1530 (size=12001) 2024-12-08T01:16:50,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/38700f0469d2479996af1cc7d1515815 2024-12-08T01:16:50,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/07e16b00e151414680849d55a4cee8cd is 50, key is test_row_0/C:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:50,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742355_1531 (size=12001) 2024-12-08T01:16:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620670832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620670832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620670832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T01:16:51,002 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:51,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-08T01:16:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:51,003 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:51,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:51,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:51,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/07e16b00e151414680849d55a4cee8cd 2024-12-08T01:16:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/8dcc1a1459824369acf5bb0658a5ada3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3 2024-12-08T01:16:51,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3, entries=200, sequenceid=50, filesize=38.6 K 2024-12-08T01:16:51,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/38700f0469d2479996af1cc7d1515815 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/38700f0469d2479996af1cc7d1515815 2024-12-08T01:16:51,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/38700f0469d2479996af1cc7d1515815, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:16:51,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/07e16b00e151414680849d55a4cee8cd as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07e16b00e151414680849d55a4cee8cd 2024-12-08T01:16:51,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07e16b00e151414680849d55a4cee8cd, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T01:16:51,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 375eeb2eab877f849984ffb3528951c4 in 848ms, sequenceid=50, compaction requested=true 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:51,022 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:51,022 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:51,023 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:51,023 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:51,023 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/A is initiating minor compaction (all files) 2024-12-08T01:16:51,023 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/B is initiating minor compaction (all files) 2024-12-08T01:16:51,023 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/B in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,023 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/A in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,023 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b29b003cf4e04cb1ae4afd8dbd7df51b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/bfc3ddb88ed4406dbf2a3600f72044e0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/38700f0469d2479996af1cc7d1515815] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=35.2 K 2024-12-08T01:16:51,023 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=99.1 K 2024-12-08T01:16:51,023 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,023 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3] 2024-12-08T01:16:51,023 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b29b003cf4e04cb1ae4afd8dbd7df51b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733620606850 2024-12-08T01:16:51,024 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea3ba98b9804c5cbab7886c81fc1441, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733620606850 2024-12-08T01:16:51,024 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc3ddb88ed4406dbf2a3600f72044e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733620606868 2024-12-08T01:16:51,024 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 38700f0469d2479996af1cc7d1515815, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620609032 2024-12-08T01:16:51,024 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting a673a6eb9e9448c292475ce9e05770f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733620606868 2024-12-08T01:16:51,024 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dcc1a1459824369acf5bb0658a5ada3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620609032 2024-12-08T01:16:51,028 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:51,029 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#B#compaction#459 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:51,029 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/ffc3b5366a1a4b2986dd2b3781bd237d is 50, key is test_row_0/B:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:51,029 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208e77a838b0cdf45b79b16a55186347343_375eeb2eab877f849984ffb3528951c4 store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:51,031 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208e77a838b0cdf45b79b16a55186347343_375eeb2eab877f849984ffb3528951c4, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:51,031 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e77a838b0cdf45b79b16a55186347343_375eeb2eab877f849984ffb3528951c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742356_1532 (size=4469) 2024-12-08T01:16:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742357_1533 (size=12104) 2024-12-08T01:16:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:51,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T01:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:51,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208607908f0dbb84489818a35af25b7fea6_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620610197/Put/seqid=0 2024-12-08T01:16:51,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742358_1534 (size=14594) 2024-12-08T01:16:51,053 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:51,060 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208607908f0dbb84489818a35af25b7fea6_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208607908f0dbb84489818a35af25b7fea6_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:51,061 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/317c8c558d5f442eb687a84171744bcb, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:51,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/317c8c558d5f442eb687a84171744bcb is 175, key is test_row_0/A:col10/1733620610197/Put/seqid=0 2024-12-08T01:16:51,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742359_1535 (size=39549) 2024-12-08T01:16:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620671071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620671072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:51,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T01:16:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620671182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620671182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T01:16:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620671341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620671342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620671342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620671392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620671392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,435 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:51,435 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/3c545eff9d8e48f79f553cc0b47dc004 is 175, key is test_row_0/A:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:51,437 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/ffc3b5366a1a4b2986dd2b3781bd237d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ffc3b5366a1a4b2986dd2b3781bd237d 2024-12-08T01:16:51,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742360_1536 (size=31058) 2024-12-08T01:16:51,444 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/B of 375eeb2eab877f849984ffb3528951c4 into ffc3b5366a1a4b2986dd2b3781bd237d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:51,444 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:51,444 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/B, priority=13, startTime=1733620611022; duration=0sec 2024-12-08T01:16:51,444 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:51,445 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:B 2024-12-08T01:16:51,445 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:51,445 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:51,445 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/C is initiating minor compaction (all files) 2024-12-08T01:16:51,445 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/C in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,445 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d3b82b087c49407e908ae2c36d641e8d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/1358d78667b1448b9d8b75dc661d4241, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07e16b00e151414680849d55a4cee8cd] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=35.2 K 2024-12-08T01:16:51,446 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b82b087c49407e908ae2c36d641e8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733620606850 2024-12-08T01:16:51,446 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 1358d78667b1448b9d8b75dc661d4241, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733620606868 2024-12-08T01:16:51,446 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e16b00e151414680849d55a4cee8cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620609032 2024-12-08T01:16:51,450 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:51,450 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/8f7914826f4a45ecac446bf063fc34b5 is 50, key is test_row_0/C:col10/1733620610173/Put/seqid=0 2024-12-08T01:16:51,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742361_1537 (size=12104) 2024-12-08T01:16:51,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T01:16:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/317c8c558d5f442eb687a84171744bcb 2024-12-08T01:16:51,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/80b16635ccf64bfe9b138543a70d6680 is 50, key is test_row_0/B:col10/1733620610197/Put/seqid=0 2024-12-08T01:16:51,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742362_1538 (size=12001) 2024-12-08T01:16:51,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T01:16:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620671702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620671702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:51,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T01:16:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:51,842 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/3c545eff9d8e48f79f553cc0b47dc004 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3c545eff9d8e48f79f553cc0b47dc004 2024-12-08T01:16:51,845 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/A of 375eeb2eab877f849984ffb3528951c4 into 3c545eff9d8e48f79f553cc0b47dc004(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:51,845 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:51,845 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/A, priority=13, startTime=1733620611022; duration=0sec 2024-12-08T01:16:51,845 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:51,845 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:A 2024-12-08T01:16:51,856 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/8f7914826f4a45ecac446bf063fc34b5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/8f7914826f4a45ecac446bf063fc34b5 2024-12-08T01:16:51,859 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/C of 375eeb2eab877f849984ffb3528951c4 into 8f7914826f4a45ecac446bf063fc34b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:51,859 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:51,859 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/C, priority=13, startTime=1733620611022; duration=0sec 2024-12-08T01:16:51,860 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:51,860 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:C 2024-12-08T01:16:51,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/80b16635ccf64bfe9b138543a70d6680 2024-12-08T01:16:51,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/07937c3054db4d12b19dcc62b3d3baf6 is 50, key is test_row_0/C:col10/1733620610197/Put/seqid=0 2024-12-08T01:16:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742363_1539 (size=12001) 2024-12-08T01:16:51,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/07937c3054db4d12b19dcc62b3d3baf6 2024-12-08T01:16:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/317c8c558d5f442eb687a84171744bcb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb 2024-12-08T01:16:51,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb, entries=200, sequenceid=70, filesize=38.6 K 2024-12-08T01:16:51,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/80b16635ccf64bfe9b138543a70d6680 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/80b16635ccf64bfe9b138543a70d6680 2024-12-08T01:16:51,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/80b16635ccf64bfe9b138543a70d6680, entries=150, sequenceid=70, filesize=11.7 K 2024-12-08T01:16:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/07937c3054db4d12b19dcc62b3d3baf6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07937c3054db4d12b19dcc62b3d3baf6 2024-12-08T01:16:51,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07937c3054db4d12b19dcc62b3d3baf6, entries=150, sequenceid=70, filesize=11.7 K 2024-12-08T01:16:51,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 375eeb2eab877f849984ffb3528951c4 in 850ms, sequenceid=70, compaction requested=false 2024-12-08T01:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:51,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T01:16:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:51,919 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T01:16:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bae6dc7534e846928ea1b4139f51d345_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620611071/Put/seqid=0 2024-12-08T01:16:51,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742364_1540 (size=12154) 2024-12-08T01:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:52,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:52,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620672239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620672240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:52,331 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bae6dc7534e846928ea1b4139f51d345_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bae6dc7534e846928ea1b4139f51d345_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/731c719012ad43ecbea7e872747305b2, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/731c719012ad43ecbea7e872747305b2 is 175, key is test_row_0/A:col10/1733620611071/Put/seqid=0 2024-12-08T01:16:52,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742365_1541 (size=30955) 2024-12-08T01:16:52,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620672352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620672352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620672352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,353 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:52,353 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_0' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,354 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620672352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620672352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620672562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620672562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,735 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/731c719012ad43ecbea7e872747305b2 2024-12-08T01:16:52,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/7b81f6b224244bc99730707efe39b3a3 is 50, key is test_row_0/B:col10/1733620611071/Put/seqid=0 2024-12-08T01:16:52,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742366_1542 (size=12001) 2024-12-08T01:16:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620672872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620672872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:53,145 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/7b81f6b224244bc99730707efe39b3a3 2024-12-08T01:16:53,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/97d49721e8c644f189294cfff3ea886b is 50, key is test_row_0/C:col10/1733620611071/Put/seqid=0 2024-12-08T01:16:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:53,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742367_1543 (size=12001) 2024-12-08T01:16:53,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620673382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:53,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620673382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:53,556 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/97d49721e8c644f189294cfff3ea886b 2024-12-08T01:16:53,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/731c719012ad43ecbea7e872747305b2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2 2024-12-08T01:16:53,563 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2, entries=150, sequenceid=85, filesize=30.2 K 2024-12-08T01:16:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/7b81f6b224244bc99730707efe39b3a3 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7b81f6b224244bc99730707efe39b3a3 2024-12-08T01:16:53,571 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7b81f6b224244bc99730707efe39b3a3, entries=150, sequenceid=85, filesize=11.7 K 2024-12-08T01:16:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/97d49721e8c644f189294cfff3ea886b as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/97d49721e8c644f189294cfff3ea886b 2024-12-08T01:16:53,576 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/97d49721e8c644f189294cfff3ea886b, entries=150, sequenceid=85, filesize=11.7 K 2024-12-08T01:16:53,579 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 375eeb2eab877f849984ffb3528951c4 in 1660ms, sequenceid=85, compaction requested=true 2024-12-08T01:16:53,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:53,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-08T01:16:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-08T01:16:53,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-08T01:16:53,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5770 sec 2024-12-08T01:16:53,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.5800 sec 2024-12-08T01:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:54,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T01:16:54,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620612231, max timestamp=1733620614363Num uniques 859; 2024-12-08T01:16:54,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620612231, max timestamp=1733620614363Num uniques 859; 2024-12-08T01:16:54,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:54,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:54,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620612231, max timestamp=1733620614363Num uniques 859; 2024-12-08T01:16:54,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:54,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:54,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081a095c226d4f4b538b8a65dedb3f383c_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742368_1544 (size=12154) 2024-12-08T01:16:54,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:54,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081a095c226d4f4b538b8a65dedb3f383c_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081a095c226d4f4b538b8a65dedb3f383c_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:54,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/bc386961d7ba4116a8d5d55a8cc38afc, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:54,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/bc386961d7ba4116a8d5d55a8cc38afc is 175, key is test_row_0/A:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742369_1545 (size=30955) 2024-12-08T01:16:54,392 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/bc386961d7ba4116a8d5d55a8cc38afc 2024-12-08T01:16:54,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/c26554abc3e84eb7b44cf0022c4cd838 is 50, key is test_row_0/B:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742370_1546 (size=12001) 2024-12-08T01:16:54,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/c26554abc3e84eb7b44cf0022c4cd838 2024-12-08T01:16:54,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620614400, max timestamp=1733620614406Num uniques 859; 2024-12-08T01:16:54,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:54,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620614400, max timestamp=1733620614406Num uniques 859; 2024-12-08T01:16:54,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:54,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733620614400, max timestamp=1733620614406Num uniques 859; 2024-12-08T01:16:54,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T01:16:54,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2f50dd1123c1461dac206bda32cbf6d7 is 50, key is test_row_0/C:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742371_1547 (size=12001) 2024-12-08T01:16:54,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2f50dd1123c1461dac206bda32cbf6d7 2024-12-08T01:16:54,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620674429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620614426, max timestamp=1733620614434Num uniques 1018; 2024-12-08T01:16:54,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T01:16:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620614426, max timestamp=1733620614434Num uniques 1018; 2024-12-08T01:16:54,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T01:16:54,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733620614426, max timestamp=1733620614434Num uniques 1018; 2024-12-08T01:16:54,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/96982154c400:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T01:16:54,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/bc386961d7ba4116a8d5d55a8cc38afc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc 2024-12-08T01:16:54,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc, entries=150, sequenceid=108, filesize=30.2 K 2024-12-08T01:16:54,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/c26554abc3e84eb7b44cf0022c4cd838 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/c26554abc3e84eb7b44cf0022c4cd838 2024-12-08T01:16:54,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/c26554abc3e84eb7b44cf0022c4cd838, entries=150, sequenceid=108, filesize=11.7 K 2024-12-08T01:16:54,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2f50dd1123c1461dac206bda32cbf6d7 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2f50dd1123c1461dac206bda32cbf6d7 2024-12-08T01:16:54,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2f50dd1123c1461dac206bda32cbf6d7, entries=150, sequenceid=108, filesize=11.7 K 2024-12-08T01:16:54,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=93.93 KB/96180 for 375eeb2eab877f849984ffb3528951c4 in 92ms, sequenceid=108, compaction requested=true 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:54,457 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:54,457 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:54,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=100.63 KB heapSize=279.84 KB 2024-12-08T01:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T01:16:54,458 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:54,458 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/A is initiating minor compaction (all files) 2024-12-08T01:16:54,458 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/A in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:54,458 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3c545eff9d8e48f79f553cc0b47dc004, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=129.4 K 2024-12-08T01:16:54,458 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:54,459 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3c545eff9d8e48f79f553cc0b47dc004, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc] 2024-12-08T01:16:54,459 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:54,459 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/B is initiating minor compaction (all files) 2024-12-08T01:16:54,459 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/B in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:54,459 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ffc3b5366a1a4b2986dd2b3781bd237d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/80b16635ccf64bfe9b138543a70d6680, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7b81f6b224244bc99730707efe39b3a3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/c26554abc3e84eb7b44cf0022c4cd838] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=47.0 K 2024-12-08T01:16:54,459 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c545eff9d8e48f79f553cc0b47dc004, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620609032 2024-12-08T01:16:54,460 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc3b5366a1a4b2986dd2b3781bd237d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620609032 2024-12-08T01:16:54,460 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317c8c558d5f442eb687a84171744bcb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733620610190 2024-12-08T01:16:54,460 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b16635ccf64bfe9b138543a70d6680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733620610190 2024-12-08T01:16:54,460 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b81f6b224244bc99730707efe39b3a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620611059 2024-12-08T01:16:54,460 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731c719012ad43ecbea7e872747305b2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620611059 2024-12-08T01:16:54,461 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting c26554abc3e84eb7b44cf0022c4cd838, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733620612239 2024-12-08T01:16:54,461 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc386961d7ba4116a8d5d55a8cc38afc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733620612239 2024-12-08T01:16:54,471 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:54,471 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b77c9354bf784a79bf653001086d742a is 50, key is test_row_0/B:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,473 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:54,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080c23bbded9a54352aa60d61c37f84c90_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620614451/Put/seqid=0 2024-12-08T01:16:54,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620674477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620674478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,485 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120855a1c59541a440a2ab594fee362d05c5_375eeb2eab877f849984ffb3528951c4 store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620674482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620674482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,487 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120855a1c59541a440a2ab594fee362d05c5_375eeb2eab877f849984ffb3528951c4, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,487 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120855a1c59541a440a2ab594fee362d05c5_375eeb2eab877f849984ffb3528951c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:54,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742372_1548 (size=12241) 2024-12-08T01:16:54,493 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b77c9354bf784a79bf653001086d742a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b77c9354bf784a79bf653001086d742a 2024-12-08T01:16:54,497 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/B of 375eeb2eab877f849984ffb3528951c4 into b77c9354bf784a79bf653001086d742a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:54,497 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:54,497 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/B, priority=12, startTime=1733620614457; duration=0sec 2024-12-08T01:16:54,497 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:54,497 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:B 2024-12-08T01:16:54,497 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:16:54,498 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:16:54,498 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/C is initiating minor compaction (all files) 2024-12-08T01:16:54,498 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/C in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:54,498 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/8f7914826f4a45ecac446bf063fc34b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07937c3054db4d12b19dcc62b3d3baf6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/97d49721e8c644f189294cfff3ea886b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2f50dd1123c1461dac206bda32cbf6d7] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=47.0 K 2024-12-08T01:16:54,498 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7914826f4a45ecac446bf063fc34b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733620609032 2024-12-08T01:16:54,498 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 07937c3054db4d12b19dcc62b3d3baf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733620610190 2024-12-08T01:16:54,499 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d49721e8c644f189294cfff3ea886b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733620611059 2024-12-08T01:16:54,500 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f50dd1123c1461dac206bda32cbf6d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733620612239 2024-12-08T01:16:54,512 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:54,513 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/7e5f4d05350144de8d3f5c8ff0ffb8b5 is 50, key is test_row_0/C:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742373_1549 (size=19774) 2024-12-08T01:16:54,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:54,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080c23bbded9a54352aa60d61c37f84c90_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080c23bbded9a54352aa60d61c37f84c90_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:54,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34c46289f2664f62abe5e298a467f67c, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:54,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34c46289f2664f62abe5e298a467f67c is 175, key is test_row_0/A:col10/1733620614451/Put/seqid=0 2024-12-08T01:16:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742374_1550 (size=4469) 2024-12-08T01:16:54,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620674542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742375_1551 (size=12241) 2024-12-08T01:16:54,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742376_1552 (size=57033) 2024-12-08T01:16:54,561 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34c46289f2664f62abe5e298a467f67c 2024-12-08T01:16:54,564 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/7e5f4d05350144de8d3f5c8ff0ffb8b5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/7e5f4d05350144de8d3f5c8ff0ffb8b5 2024-12-08T01:16:54,569 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/C of 375eeb2eab877f849984ffb3528951c4 into 7e5f4d05350144de8d3f5c8ff0ffb8b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:54,569 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:54,569 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/C, priority=12, startTime=1733620614457; duration=0sec 2024-12-08T01:16:54,569 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:54,569 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:C 2024-12-08T01:16:54,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/a127abe0643745e2b7d6ffe9cf91d679 is 50, key is test_row_0/B:col10/1733620614451/Put/seqid=0 2024-12-08T01:16:54,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742377_1553 (size=12151) 2024-12-08T01:16:54,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/a127abe0643745e2b7d6ffe9cf91d679 2024-12-08T01:16:54,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/37174bf6563d488b991d311e44b8d4c9 is 50, key is test_row_0/C:col10/1733620614451/Put/seqid=0 2024-12-08T01:16:54,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620674592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620674592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620674592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620674592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742378_1554 (size=12151) 2024-12-08T01:16:54,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620674752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620674801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620674801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620674801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620674802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:54,912 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:16:54,944 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#A#compaction#472 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:54,944 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/814dc4defb6a4954895434044e734dd1 is 175, key is test_row_0/A:col10/1733620614365/Put/seqid=0 2024-12-08T01:16:54,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742379_1555 (size=31195) 2024-12-08T01:16:55,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/37174bf6563d488b991d311e44b8d4c9 2024-12-08T01:16:55,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34c46289f2664f62abe5e298a467f67c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c 2024-12-08T01:16:55,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c, entries=300, sequenceid=156, filesize=55.7 K 2024-12-08T01:16:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/a127abe0643745e2b7d6ffe9cf91d679 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a127abe0643745e2b7d6ffe9cf91d679 2024-12-08T01:16:55,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a127abe0643745e2b7d6ffe9cf91d679, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T01:16:55,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/37174bf6563d488b991d311e44b8d4c9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/37174bf6563d488b991d311e44b8d4c9 2024-12-08T01:16:55,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/37174bf6563d488b991d311e44b8d4c9, entries=150, sequenceid=156, filesize=11.9 K 2024-12-08T01:16:55,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~279.80 KB/286512, currentSize=80.51 KB/82440 for 375eeb2eab877f849984ffb3528951c4 in 557ms, sequenceid=156, compaction requested=false 2024-12-08T01:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:55,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:16:55,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:55,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:55,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:55,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:55,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:55,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:55,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088f561677f78c4493ab54815f394dde46_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:55,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742380_1556 (size=12304) 2024-12-08T01:16:55,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:55,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088f561677f78c4493ab54815f394dde46_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088f561677f78c4493ab54815f394dde46_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:55,076 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/dabb163d23a24d099fb6e6fa3230c73a, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:55,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/dabb163d23a24d099fb6e6fa3230c73a is 175, key is test_row_0/A:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:55,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742381_1557 (size=31105) 2024-12-08T01:16:55,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620675112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620675112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620675112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620675112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620675112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T01:16:55,162 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:55,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-08T01:16:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:55,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:55,165 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:55,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620675221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:55,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,394 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/814dc4defb6a4954895434044e734dd1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/814dc4defb6a4954895434044e734dd1 2024-12-08T01:16:55,398 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/A of 375eeb2eab877f849984ffb3528951c4 into 814dc4defb6a4954895434044e734dd1(size=30.5 K), total size for store is 86.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:55,398 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:55,398 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/A, priority=12, startTime=1733620614457; duration=0sec 2024-12-08T01:16:55,398 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:55,398 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:A 2024-12-08T01:16:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620675432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:55,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:55,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/dabb163d23a24d099fb6e6fa3230c73a 2024-12-08T01:16:55,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/7d009fa70dc047e2bdf291a2f86ddc4f is 50, key is test_row_0/B:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:55,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742382_1558 (size=12151) 2024-12-08T01:16:55,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:55,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:55,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620675622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620675622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620675622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620675622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620675742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:55,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:55,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:55,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:55,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/7d009fa70dc047e2bdf291a2f86ddc4f 2024-12-08T01:16:55,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9403426e2847468dbbe75a3db4ed15fc is 50, key is test_row_0/C:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:55,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742383_1559 (size=12151) 2024-12-08T01:16:55,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:55,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:55,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:56,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620676252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:56,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9403426e2847468dbbe75a3db4ed15fc 2024-12-08T01:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/dabb163d23a24d099fb6e6fa3230c73a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a 2024-12-08T01:16:56,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a, entries=150, sequenceid=174, filesize=30.4 K 2024-12-08T01:16:56,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/7d009fa70dc047e2bdf291a2f86ddc4f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7d009fa70dc047e2bdf291a2f86ddc4f 2024-12-08T01:16:56,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7d009fa70dc047e2bdf291a2f86ddc4f, entries=150, sequenceid=174, filesize=11.9 K 2024-12-08T01:16:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9403426e2847468dbbe75a3db4ed15fc as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9403426e2847468dbbe75a3db4ed15fc 2024-12-08T01:16:56,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9403426e2847468dbbe75a3db4ed15fc, entries=150, sequenceid=174, filesize=11.9 K 2024-12-08T01:16:56,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 375eeb2eab877f849984ffb3528951c4 in 1249ms, sequenceid=174, compaction requested=true 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:56,315 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:56,315 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:56,316 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:56,316 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/A is initiating minor compaction (all files) 2024-12-08T01:16:56,316 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:56,316 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/A in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,316 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/B is initiating minor compaction (all files) 2024-12-08T01:16:56,316 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/B in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,316 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/814dc4defb6a4954895434044e734dd1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=116.5 K 2024-12-08T01:16:56,316 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b77c9354bf784a79bf653001086d742a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a127abe0643745e2b7d6ffe9cf91d679, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7d009fa70dc047e2bdf291a2f86ddc4f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=35.7 K 2024-12-08T01:16:56,316 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,316 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/814dc4defb6a4954895434044e734dd1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a] 2024-12-08T01:16:56,317 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b77c9354bf784a79bf653001086d742a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733620612239 2024-12-08T01:16:56,317 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814dc4defb6a4954895434044e734dd1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733620612239 2024-12-08T01:16:56,317 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting a127abe0643745e2b7d6ffe9cf91d679, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620614434 2024-12-08T01:16:56,317 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c46289f2664f62abe5e298a467f67c, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620614434 2024-12-08T01:16:56,317 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d009fa70dc047e2bdf291a2f86ddc4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620614476 2024-12-08T01:16:56,317 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting dabb163d23a24d099fb6e6fa3230c73a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620614476 2024-12-08T01:16:56,322 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:56,323 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/efee0627f2c24ec8b682a45d4fecae08 is 50, key is test_row_0/B:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:56,324 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:56,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742384_1560 (size=12493) 2024-12-08T01:16:56,335 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208ac656ad4e4aa4f1ea7df0222d1cc5820_375eeb2eab877f849984ffb3528951c4 store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:56,336 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208ac656ad4e4aa4f1ea7df0222d1cc5820_375eeb2eab877f849984ffb3528951c4, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:56,336 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ac656ad4e4aa4f1ea7df0222d1cc5820_375eeb2eab877f849984ffb3528951c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:56,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742385_1561 (size=4469) 2024-12-08T01:16:56,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T01:16:56,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,385 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:16:56,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:56,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:56,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:56,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:56,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:56,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120885d3a87e64e143f480149635d806ea08_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620615089/Put/seqid=0 2024-12-08T01:16:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742386_1562 (size=12304) 2024-12-08T01:16:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:56,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:56,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620676643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620676645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620676646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620676647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,733 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/efee0627f2c24ec8b682a45d4fecae08 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/efee0627f2c24ec8b682a45d4fecae08 2024-12-08T01:16:56,737 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/B of 375eeb2eab877f849984ffb3528951c4 into efee0627f2c24ec8b682a45d4fecae08(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:56,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:56,737 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/B, priority=13, startTime=1733620616315; duration=0sec 2024-12-08T01:16:56,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:56,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:B 2024-12-08T01:16:56,737 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:56,738 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:56,738 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/C is initiating minor compaction (all files) 2024-12-08T01:16:56,738 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/C in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:56,738 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/7e5f4d05350144de8d3f5c8ff0ffb8b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/37174bf6563d488b991d311e44b8d4c9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9403426e2847468dbbe75a3db4ed15fc] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=35.7 K 2024-12-08T01:16:56,738 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5f4d05350144de8d3f5c8ff0ffb8b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733620612239 2024-12-08T01:16:56,738 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 37174bf6563d488b991d311e44b8d4c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733620614434 2024-12-08T01:16:56,738 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 9403426e2847468dbbe75a3db4ed15fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620614476 2024-12-08T01:16:56,743 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:56,744 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9699795173484a098bd67871bd3d409a is 50, key is test_row_0/C:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:56,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742387_1563 (size=12493) 2024-12-08T01:16:56,748 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#A#compaction#481 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:56,748 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/4304e0263bfc472b9c3867881ff13984 is 175, key is test_row_0/A:col10/1733620615064/Put/seqid=0 2024-12-08T01:16:56,751 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9699795173484a098bd67871bd3d409a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9699795173484a098bd67871bd3d409a 2024-12-08T01:16:56,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620676752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620676752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620676752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620676752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742388_1564 (size=31447) 2024-12-08T01:16:56,759 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/C of 375eeb2eab877f849984ffb3528951c4 into 9699795173484a098bd67871bd3d409a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:56,759 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:56,759 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/C, priority=13, startTime=1733620616315; duration=0sec 2024-12-08T01:16:56,760 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:56,760 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:C 2024-12-08T01:16:56,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:56,796 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120885d3a87e64e143f480149635d806ea08_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120885d3a87e64e143f480149635d806ea08_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/11393aff1e7d449d8a8dc918534cca03, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/11393aff1e7d449d8a8dc918534cca03 is 175, key is test_row_0/A:col10/1733620615089/Put/seqid=0 2024-12-08T01:16:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742389_1565 (size=31105) 2024-12-08T01:16:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620676961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620676962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620676962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620676962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,155 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/4304e0263bfc472b9c3867881ff13984 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/4304e0263bfc472b9c3867881ff13984 2024-12-08T01:16:57,158 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/A of 375eeb2eab877f849984ffb3528951c4 into 4304e0263bfc472b9c3867881ff13984(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:57,158 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:57,158 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/A, priority=13, startTime=1733620616315; duration=0sec 2024-12-08T01:16:57,158 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:57,158 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:A 2024-12-08T01:16:57,201 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/11393aff1e7d449d8a8dc918534cca03 2024-12-08T01:16:57,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/0a4190e15e5a424697be68ceb55ce9d9 is 50, key is test_row_0/B:col10/1733620615089/Put/seqid=0 2024-12-08T01:16:57,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742390_1566 (size=12151) 2024-12-08T01:16:57,210 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/0a4190e15e5a424697be68ceb55ce9d9 2024-12-08T01:16:57,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9d978fb31afd4bc7bc8dd6561fc9d3fb is 50, key is test_row_0/C:col10/1733620615089/Put/seqid=0 2024-12-08T01:16:57,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742391_1567 (size=12151) 2024-12-08T01:16:57,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620677272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620677272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620677272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,274 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620677272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620677272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:57,620 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9d978fb31afd4bc7bc8dd6561fc9d3fb 2024-12-08T01:16:57,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/11393aff1e7d449d8a8dc918534cca03 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03 2024-12-08T01:16:57,625 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03, entries=150, sequenceid=191, filesize=30.4 K 2024-12-08T01:16:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/0a4190e15e5a424697be68ceb55ce9d9 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/0a4190e15e5a424697be68ceb55ce9d9 2024-12-08T01:16:57,629 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/0a4190e15e5a424697be68ceb55ce9d9, entries=150, sequenceid=191, filesize=11.9 K 2024-12-08T01:16:57,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/9d978fb31afd4bc7bc8dd6561fc9d3fb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9d978fb31afd4bc7bc8dd6561fc9d3fb 2024-12-08T01:16:57,632 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9d978fb31afd4bc7bc8dd6561fc9d3fb, entries=150, sequenceid=191, filesize=11.9 K 2024-12-08T01:16:57,632 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 375eeb2eab877f849984ffb3528951c4 in 1247ms, sequenceid=191, compaction requested=false 2024-12-08T01:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-08T01:16:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-08T01:16:57,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-08T01:16:57,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-12-08T01:16:57,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.4720 sec 2024-12-08T01:16:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:57,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:57,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d76736ea37ce43139b4b4a729283bfcc_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620677792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620677793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742392_1568 (size=12304) 2024-12-08T01:16:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620677794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620677795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620677901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620677901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620677902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620677902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620678113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620678113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620678113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620678113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,198 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:58,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d76736ea37ce43139b4b4a729283bfcc_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d76736ea37ce43139b4b4a729283bfcc_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:58,200 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/0c85bc21af904b02b68598892182b222, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:58,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/0c85bc21af904b02b68598892182b222 is 175, key is test_row_0/A:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:58,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742393_1569 (size=31105) 2024-12-08T01:16:58,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620678422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620678422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620678422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620678422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,604 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/0c85bc21af904b02b68598892182b222 2024-12-08T01:16:58,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/3af6aabeda294969880eb1efb3404b0a is 50, key is test_row_0/B:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:58,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742394_1570 (size=12151) 2024-12-08T01:16:58,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620678932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620678932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620678932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:58,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620678932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/3af6aabeda294969880eb1efb3404b0a 2024-12-08T01:16:59,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/40187649a4744676a892837ebfb7c331 is 50, key is test_row_0/C:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:59,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742395_1571 (size=12151) 2024-12-08T01:16:59,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/40187649a4744676a892837ebfb7c331 2024-12-08T01:16:59,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/0c85bc21af904b02b68598892182b222 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222 2024-12-08T01:16:59,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222, entries=150, sequenceid=211, filesize=30.4 K 2024-12-08T01:16:59,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/3af6aabeda294969880eb1efb3404b0a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/3af6aabeda294969880eb1efb3404b0a 2024-12-08T01:16:59,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/3af6aabeda294969880eb1efb3404b0a, entries=150, sequenceid=211, filesize=11.9 K 2024-12-08T01:16:59,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/40187649a4744676a892837ebfb7c331 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/40187649a4744676a892837ebfb7c331 2024-12-08T01:16:59,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/40187649a4744676a892837ebfb7c331, entries=150, sequenceid=211, filesize=11.9 K 2024-12-08T01:16:59,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 375eeb2eab877f849984ffb3528951c4 in 1249ms, sequenceid=211, compaction requested=true 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:16:59,032 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:59,032 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:16:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:59,033 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:59,033 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:59,033 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/A is initiating minor compaction (all files) 2024-12-08T01:16:59,033 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/B is initiating minor compaction (all files) 2024-12-08T01:16:59,033 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/A in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,033 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/B in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,033 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/4304e0263bfc472b9c3867881ff13984, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=91.5 K 2024-12-08T01:16:59,033 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/efee0627f2c24ec8b682a45d4fecae08, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/0a4190e15e5a424697be68ceb55ce9d9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/3af6aabeda294969880eb1efb3404b0a] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=35.9 K 2024-12-08T01:16:59,033 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,033 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/4304e0263bfc472b9c3867881ff13984, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222] 2024-12-08T01:16:59,034 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting efee0627f2c24ec8b682a45d4fecae08, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620614476 2024-12-08T01:16:59,034 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4304e0263bfc472b9c3867881ff13984, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620614476 2024-12-08T01:16:59,034 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11393aff1e7d449d8a8dc918534cca03, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733620615089 2024-12-08T01:16:59,035 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4190e15e5a424697be68ceb55ce9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733620615089 2024-12-08T01:16:59,035 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c85bc21af904b02b68598892182b222, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733620616635 2024-12-08T01:16:59,035 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af6aabeda294969880eb1efb3404b0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733620616635 2024-12-08T01:16:59,040 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:59,041 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#B#compaction#489 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:59,041 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/de96d659ef3d49d2ab358360564afc78 is 50, key is test_row_0/B:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:59,041 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208fb0e74f2513d4d3ab4b83c59ec1ac35f_375eeb2eab877f849984ffb3528951c4 store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:59,043 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208fb0e74f2513d4d3ab4b83c59ec1ac35f_375eeb2eab877f849984ffb3528951c4, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:59,043 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fb0e74f2513d4d3ab4b83c59ec1ac35f_375eeb2eab877f849984ffb3528951c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742396_1572 (size=12595) 2024-12-08T01:16:59,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742397_1573 (size=4469) 2024-12-08T01:16:59,050 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/de96d659ef3d49d2ab358360564afc78 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/de96d659ef3d49d2ab358360564afc78 2024-12-08T01:16:59,050 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#A#compaction#490 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:59,051 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/856f8351d63043bc8451b2724a088650 is 175, key is test_row_0/A:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:59,054 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/B of 375eeb2eab877f849984ffb3528951c4 into de96d659ef3d49d2ab358360564afc78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:59,054 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:59,054 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/B, priority=13, startTime=1733620619032; duration=0sec 2024-12-08T01:16:59,054 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:16:59,054 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:B 2024-12-08T01:16:59,054 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:16:59,055 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:16:59,055 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/C is initiating minor compaction (all files) 2024-12-08T01:16:59,055 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/C in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,055 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9699795173484a098bd67871bd3d409a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9d978fb31afd4bc7bc8dd6561fc9d3fb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/40187649a4744676a892837ebfb7c331] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=35.9 K 2024-12-08T01:16:59,056 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 9699795173484a098bd67871bd3d409a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733620614476 2024-12-08T01:16:59,056 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d978fb31afd4bc7bc8dd6561fc9d3fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733620615089 2024-12-08T01:16:59,057 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 40187649a4744676a892837ebfb7c331, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733620616635 2024-12-08T01:16:59,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742398_1574 (size=31549) 2024-12-08T01:16:59,062 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:16:59,062 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/ed3e51db2ef2437fb463c8058cbaf7e2 is 50, key is test_row_0/C:col10/1733620617782/Put/seqid=0 2024-12-08T01:16:59,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742399_1575 (size=12595) 2024-12-08T01:16:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:59,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:16:59,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:16:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:16:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:16:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:16:59,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089b13c5f17a0148eea72ff599411f71ba_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620617785/Put/seqid=0 2024-12-08T01:16:59,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742400_1576 (size=14794) 2024-12-08T01:16:59,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620679331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T01:16:59,342 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:16:59,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:16:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-08T01:16:59,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:16:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:16:59,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:16:59,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:16:59,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620679442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:16:59,461 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/856f8351d63043bc8451b2724a088650 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/856f8351d63043bc8451b2724a088650 2024-12-08T01:16:59,464 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/A of 375eeb2eab877f849984ffb3528951c4 into 856f8351d63043bc8451b2724a088650(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:59,464 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:59,464 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/A, priority=13, startTime=1733620619032; duration=0sec 2024-12-08T01:16:59,465 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:59,465 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:A 2024-12-08T01:16:59,473 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/ed3e51db2ef2437fb463c8058cbaf7e2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/ed3e51db2ef2437fb463c8058cbaf7e2 2024-12-08T01:16:59,476 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/C of 375eeb2eab877f849984ffb3528951c4 into ed3e51db2ef2437fb463c8058cbaf7e2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:16:59,476 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:16:59,476 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/C, priority=13, startTime=1733620619032; duration=0sec 2024-12-08T01:16:59,476 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:16:59,476 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:C 2024-12-08T01:16:59,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:16:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:16:59,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:59,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620679652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:16:59,706 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:16:59,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089b13c5f17a0148eea72ff599411f71ba_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089b13c5f17a0148eea72ff599411f71ba_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:16:59,710 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/381e4af9f1804085a8a26a30b946057c, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:16:59,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/381e4af9f1804085a8a26a30b946057c is 175, key is test_row_0/A:col10/1733620617785/Put/seqid=0 2024-12-08T01:16:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742401_1577 (size=39749) 2024-12-08T01:16:59,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:16:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620679941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620679941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,943 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,944 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620679952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620679952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,953 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:59,953 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:16:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:16:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:16:59,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:16:59,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:16:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620679964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:16:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:16:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:16:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:17:00,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:17:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/381e4af9f1804085a8a26a30b946057c 2024-12-08T01:17:00,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/431971fa339248e684bd236d5602ee31 is 50, key is test_row_0/B:col10/1733620617785/Put/seqid=0 2024-12-08T01:17:00,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742402_1578 (size=12151) 2024-12-08T01:17:00,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/431971fa339248e684bd236d5602ee31 2024-12-08T01:17:00,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/fbfaf519e7a546fba2943d62c8fde458 is 50, key is test_row_0/C:col10/1733620617785/Put/seqid=0 2024-12-08T01:17:00,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742403_1579 (size=12151) 2024-12-08T01:17:00,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:17:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:00,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620680472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:17:00,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/fbfaf519e7a546fba2943d62c8fde458 2024-12-08T01:17:00,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/381e4af9f1804085a8a26a30b946057c as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c 2024-12-08T01:17:00,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c, entries=200, sequenceid=228, filesize=38.8 K 2024-12-08T01:17:00,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/431971fa339248e684bd236d5602ee31 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/431971fa339248e684bd236d5602ee31 2024-12-08T01:17:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/431971fa339248e684bd236d5602ee31, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T01:17:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/fbfaf519e7a546fba2943d62c8fde458 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/fbfaf519e7a546fba2943d62c8fde458 2024-12-08T01:17:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/fbfaf519e7a546fba2943d62c8fde458, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T01:17:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 375eeb2eab877f849984ffb3528951c4 in 1268ms, sequenceid=228, compaction requested=false 2024-12-08T01:17:00,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T01:17:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:00,565 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T01:17:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120813f72ea03679439ba18d242ea3c1db52_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_1/A:col10/1733620619330/Put/seqid=0 2024-12-08T01:17:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742404_1580 (size=9814) 2024-12-08T01:17:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,995 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120813f72ea03679439ba18d242ea3c1db52_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120813f72ea03679439ba18d242ea3c1db52_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/51ce8858bf584a9281055d24b677e820, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/51ce8858bf584a9281055d24b677e820 is 175, key is test_row_1/A:col10/1733620619330/Put/seqid=0 2024-12-08T01:17:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742405_1581 (size=22461) 2024-12-08T01:17:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,401 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/51ce8858bf584a9281055d24b677e820 2024-12-08T01:17:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/ec51e41361be49de83d1d96614bcc985 is 50, key is test_row_1/B:col10/1733620619330/Put/seqid=0 2024-12-08T01:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742406_1582 (size=9757) 2024-12-08T01:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:17:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:01,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620681531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:01,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620681642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:01,812 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/ec51e41361be49de83d1d96614bcc985 2024-12-08T01:17:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/f13d3efcbce445ea92c4a1ea35e3f54a is 50, key is test_row_1/C:col10/1733620619330/Put/seqid=0 2024-12-08T01:17:01,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742407_1583 (size=9757) 2024-12-08T01:17:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620681852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:01,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620681952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,955 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:17:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620681961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,964 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:17:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:01,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620681971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620681972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:01,974 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_2' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:01,974 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:17:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:02,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620682162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:02,220 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/f13d3efcbce445ea92c4a1ea35e3f54a 2024-12-08T01:17:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/51ce8858bf584a9281055d24b677e820 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820 2024-12-08T01:17:02,226 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820, entries=100, sequenceid=246, filesize=21.9 K 2024-12-08T01:17:02,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/ec51e41361be49de83d1d96614bcc985 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ec51e41361be49de83d1d96614bcc985 2024-12-08T01:17:02,229 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ec51e41361be49de83d1d96614bcc985, entries=100, sequenceid=246, filesize=9.5 K 2024-12-08T01:17:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/f13d3efcbce445ea92c4a1ea35e3f54a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/f13d3efcbce445ea92c4a1ea35e3f54a 2024-12-08T01:17:02,232 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/f13d3efcbce445ea92c4a1ea35e3f54a, entries=100, sequenceid=246, filesize=9.5 K 2024-12-08T01:17:02,233 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 375eeb2eab877f849984ffb3528951c4 in 1668ms, sequenceid=246, compaction requested=true 2024-12-08T01:17:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-08T01:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-08T01:17:02,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-08T01:17:02,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8890 sec 2024-12-08T01:17:02,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.8930 sec 2024-12-08T01:17:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:02,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:17:02,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:17:02,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:02,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:17:02,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:02,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:17:02,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:02,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084376b0ba87fa4851863b5cca939f501d_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:02,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742408_1584 (size=12454) 2024-12-08T01:17:02,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620682718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:02,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620682832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:03,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620683042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:03,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:03,087 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084376b0ba87fa4851863b5cca939f501d_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084376b0ba87fa4851863b5cca939f501d_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:03,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/3eebfbcb55b24be8a5977f75511367ab, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:03,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/3eebfbcb55b24be8a5977f75511367ab is 175, key is test_row_0/A:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:03,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742409_1585 (size=31255) 2024-12-08T01:17:03,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/3eebfbcb55b24be8a5977f75511367ab 2024-12-08T01:17:03,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/4010cd83cc0e46e2a682bc4a9fba373f is 50, key is test_row_0/B:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:03,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742410_1586 (size=12301) 2024-12-08T01:17:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620683352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:03,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/4010cd83cc0e46e2a682bc4a9fba373f 2024-12-08T01:17:03,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/edd5c037e3f04c4d8d0c16154ec8cfc5 is 50, key is test_row_0/C:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:03,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742411_1587 (size=12301) 2024-12-08T01:17:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T01:17:03,511 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:17:03,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T01:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-08T01:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:03,513 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T01:17:03,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T01:17:03,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T01:17:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:03,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T01:17:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:03,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T01:17:03,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:03,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:17:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:03,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620683862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:03,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/edd5c037e3f04c4d8d0c16154ec8cfc5 2024-12-08T01:17:03,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/3eebfbcb55b24be8a5977f75511367ab as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab 2024-12-08T01:17:03,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab, entries=150, sequenceid=263, filesize=30.5 K 2024-12-08T01:17:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/4010cd83cc0e46e2a682bc4a9fba373f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/4010cd83cc0e46e2a682bc4a9fba373f 2024-12-08T01:17:03,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/4010cd83cc0e46e2a682bc4a9fba373f, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:17:03,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/edd5c037e3f04c4d8d0c16154ec8cfc5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/edd5c037e3f04c4d8d0c16154ec8cfc5 2024-12-08T01:17:03,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/edd5c037e3f04c4d8d0c16154ec8cfc5, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T01:17:03,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 375eeb2eab877f849984ffb3528951c4 in 1249ms, sequenceid=263, compaction requested=true 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:03,924 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:17:03,924 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:17:03,925 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46804 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:17:03,925 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/B is initiating minor compaction (all files) 2024-12-08T01:17:03,925 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/B in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,925 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/de96d659ef3d49d2ab358360564afc78, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/431971fa339248e684bd236d5602ee31, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ec51e41361be49de83d1d96614bcc985, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/4010cd83cc0e46e2a682bc4a9fba373f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=45.7 K 2024-12-08T01:17:03,925 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting de96d659ef3d49d2ab358360564afc78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733620616635 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 431971fa339248e684bd236d5602ee31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620617785 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting ec51e41361be49de83d1d96614bcc985, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733620619324 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/A is initiating minor compaction (all files) 2024-12-08T01:17:03,926 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/A in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,926 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/856f8351d63043bc8451b2724a088650, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=122.1 K 2024-12-08T01:17:03,926 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/856f8351d63043bc8451b2724a088650, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab] 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856f8351d63043bc8451b2724a088650, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733620616635 2024-12-08T01:17:03,926 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 4010cd83cc0e46e2a682bc4a9fba373f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620621526 2024-12-08T01:17:03,927 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381e4af9f1804085a8a26a30b946057c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620617785 2024-12-08T01:17:03,928 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51ce8858bf584a9281055d24b677e820, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733620619324 2024-12-08T01:17:03,929 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eebfbcb55b24be8a5977f75511367ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620621526 2024-12-08T01:17:03,937 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:03,939 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:17:03,939 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/e841aac9d1ef4d9988fa9b89e33fdf48 is 50, key is test_row_0/B:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:03,941 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120858519e152cdc49d6b86ccec30dbd798d_375eeb2eab877f849984ffb3528951c4 store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:03,942 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120858519e152cdc49d6b86ccec30dbd798d_375eeb2eab877f849984ffb3528951c4, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:03,942 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120858519e152cdc49d6b86ccec30dbd798d_375eeb2eab877f849984ffb3528951c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:03,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742413_1589 (size=12881) 2024-12-08T01:17:03,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742412_1588 (size=4469) 2024-12-08T01:17:03,964 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#A#compaction#502 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:17:03,964 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/58357a097b1b4bbfa83c102ebfcdd9c5 is 175, key is test_row_0/A:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:03,965 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/e841aac9d1ef4d9988fa9b89e33fdf48 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e841aac9d1ef4d9988fa9b89e33fdf48 2024-12-08T01:17:03,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742414_1590 (size=31835) 2024-12-08T01:17:03,969 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/B of 375eeb2eab877f849984ffb3528951c4 into e841aac9d1ef4d9988fa9b89e33fdf48(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:17:03,969 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:03,969 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/B, priority=12, startTime=1733620623924; duration=0sec 2024-12-08T01:17:03,970 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:17:03,970 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:B 2024-12-08T01:17:03,970 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:17:03,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44843 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T01:17:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,972 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T01:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:17:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:03,973 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46804 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T01:17:03,973 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/C is initiating minor compaction (all files) 2024-12-08T01:17:03,973 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/C in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:03,973 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/ed3e51db2ef2437fb463c8058cbaf7e2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/fbfaf519e7a546fba2943d62c8fde458, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/f13d3efcbce445ea92c4a1ea35e3f54a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/edd5c037e3f04c4d8d0c16154ec8cfc5] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=45.7 K 2024-12-08T01:17:03,973 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/58357a097b1b4bbfa83c102ebfcdd9c5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/58357a097b1b4bbfa83c102ebfcdd9c5 2024-12-08T01:17:03,973 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3e51db2ef2437fb463c8058cbaf7e2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733620616635 2024-12-08T01:17:03,973 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting fbfaf519e7a546fba2943d62c8fde458, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733620617785 2024-12-08T01:17:03,974 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting f13d3efcbce445ea92c4a1ea35e3f54a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733620619324 2024-12-08T01:17:03,974 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting edd5c037e3f04c4d8d0c16154ec8cfc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620621526 2024-12-08T01:17:03,978 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/A of 375eeb2eab877f849984ffb3528951c4 into 58357a097b1b4bbfa83c102ebfcdd9c5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:17:03,978 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:03,978 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/A, priority=12, startTime=1733620623924; duration=0sec 2024-12-08T01:17:03,978 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:03,978 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:A 2024-12-08T01:17:03,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086bb48c01e1f647a1b09453fb774feef0_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620622700/Put/seqid=0 2024-12-08T01:17:03,983 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:17:03,983 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/beb7ff5aef4e46b095a1e4c8a22964d1 is 50, key is test_row_0/C:col10/1733620621530/Put/seqid=0 2024-12-08T01:17:03,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742416_1592 (size=12881) 2024-12-08T01:17:03,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742415_1591 (size=12454) 2024-12-08T01:17:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:04,390 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/beb7ff5aef4e46b095a1e4c8a22964d1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/beb7ff5aef4e46b095a1e4c8a22964d1 2024-12-08T01:17:04,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:04,393 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086bb48c01e1f647a1b09453fb774feef0_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086bb48c01e1f647a1b09453fb774feef0_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:04,394 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/C of 375eeb2eab877f849984ffb3528951c4 into beb7ff5aef4e46b095a1e4c8a22964d1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:17:04,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:04,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/31567d7faf5e4e4b9a1a8a19abd8f41e, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:04,394 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/C, priority=12, startTime=1733620623924; duration=0sec 2024-12-08T01:17:04,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:04,394 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:C 2024-12-08T01:17:04,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/31567d7faf5e4e4b9a1a8a19abd8f41e is 175, key is test_row_0/A:col10/1733620622700/Put/seqid=0 2024-12-08T01:17:04,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742417_1593 (size=31255) 2024-12-08T01:17:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:04,797 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/31567d7faf5e4e4b9a1a8a19abd8f41e 2024-12-08T01:17:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/e503788aa62f4ccf9b098bcda1a4891a is 50, key is test_row_0/B:col10/1733620622700/Put/seqid=0 2024-12-08T01:17:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742418_1594 (size=12301) 2024-12-08T01:17:04,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. as already flushing 2024-12-08T01:17:04,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620684957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620685073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:05,247 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/e503788aa62f4ccf9b098bcda1a4891a 2024-12-08T01:17:05,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/df4d11dfaea44f70a4bd42f82e1dfaba is 50, key is test_row_0/C:col10/1733620622700/Put/seqid=0 2024-12-08T01:17:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620685281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:05,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742419_1595 (size=12301) 2024-12-08T01:17:05,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620685592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:05,707 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/df4d11dfaea44f70a4bd42f82e1dfaba 2024-12-08T01:17:05,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/31567d7faf5e4e4b9a1a8a19abd8f41e as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e 2024-12-08T01:17:05,714 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e, entries=150, sequenceid=279, filesize=30.5 K 2024-12-08T01:17:05,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/e503788aa62f4ccf9b098bcda1a4891a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e503788aa62f4ccf9b098bcda1a4891a 2024-12-08T01:17:05,728 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e503788aa62f4ccf9b098bcda1a4891a, entries=150, sequenceid=279, filesize=12.0 K 2024-12-08T01:17:05,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/df4d11dfaea44f70a4bd42f82e1dfaba as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/df4d11dfaea44f70a4bd42f82e1dfaba 2024-12-08T01:17:05,731 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/df4d11dfaea44f70a4bd42f82e1dfaba, entries=150, sequenceid=279, filesize=12.0 K 2024-12-08T01:17:05,732 INFO [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 375eeb2eab877f849984ffb3528951c4 in 1761ms, sequenceid=279, compaction requested=false 2024-12-08T01:17:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/96982154c400:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-08T01:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-08T01:17:05,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-08T01:17:05,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2190 sec 2024-12-08T01:17:05,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2230 sec 2024-12-08T01:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:05,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T01:17:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:17:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:17:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:17:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:06,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208045d2842b25d49c4a04433cad904b4ae_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:06,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742420_1596 (size=12454) 2024-12-08T01:17:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620686009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620686010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620686012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52616 deadline: 1733620686012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,015 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620686105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620686121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620686121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620686122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620686332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620686332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620686332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:06,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208045d2842b25d49c4a04433cad904b4ae_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208045d2842b25d49c4a04433cad904b4ae_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:06,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/79f5f4f8e94d4581b01394c8188ae5bb, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:06,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/79f5f4f8e94d4581b01394c8188ae5bb is 175, key is test_row_0/A:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:06,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742421_1597 (size=31255) 2024-12-08T01:17:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620686642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620686642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620686642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:06,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/79f5f4f8e94d4581b01394c8188ae5bb 2024-12-08T01:17:06,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b6b733fcb03c47dca3737c7b25ae0a1d is 50, key is test_row_0/B:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:06,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742422_1598 (size=12301) 2024-12-08T01:17:06,845 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-12-08T01:17:06,845 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:06,845 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,845 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:06,846 INFO [Thread-2349 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2349. 2024-12-08T01:17:06,846 DEBUG [Thread-2349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:06,846 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,846 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:06,846 INFO [Thread-2353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2353. 2024-12-08T01:17:06,846 DEBUG [Thread-2353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:06,846 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,846 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:06,848 INFO [Thread-2351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2351. 2024-12-08T01:17:06,848 DEBUG [Thread-2351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:06,848 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,848 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:06,848 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-12-08T01:17:06,848 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:06,848 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,848 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:06,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:07,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52578 deadline: 1733620687112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:07,112 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 96982154c400,44843,1733620436630 for 'test_row_1' in 375eeb2eab877f849984ffb3528951c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T01:17:07,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:07,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:07,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:07,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T01:17:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52590 deadline: 1733620687152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52562 deadline: 1733620687152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44843 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52592 deadline: 1733620687152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 , the old value is region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., hostname=96982154c400,44843,1733620436630, seqNum=5 because the exception is null or not the one we care about 2024-12-08T01:17:07,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b6b733fcb03c47dca3737c7b25ae0a1d 2024-12-08T01:17:07,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2bcf277411184882b51bfdfdb865136f is 50, key is test_row_0/C:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:07,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742423_1599 (size=12301) 2024-12-08T01:17:07,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2bcf277411184882b51bfdfdb865136f 2024-12-08T01:17:07,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/79f5f4f8e94d4581b01394c8188ae5bb as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb 2024-12-08T01:17:07,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb, entries=150, sequenceid=299, filesize=30.5 K 2024-12-08T01:17:07,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/b6b733fcb03c47dca3737c7b25ae0a1d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b6b733fcb03c47dca3737c7b25ae0a1d 2024-12-08T01:17:07,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b6b733fcb03c47dca3737c7b25ae0a1d, entries=150, sequenceid=299, filesize=12.0 K 2024-12-08T01:17:07,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2bcf277411184882b51bfdfdb865136f as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2bcf277411184882b51bfdfdb865136f 2024-12-08T01:17:07,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2bcf277411184882b51bfdfdb865136f, entries=150, sequenceid=299, filesize=12.0 K 2024-12-08T01:17:07,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 375eeb2eab877f849984ffb3528951c4 in 1666ms, sequenceid=299, compaction requested=true 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:07,660 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 375eeb2eab877f849984ffb3528951c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T01:17:07,660 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:17:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/A is initiating minor compaction (all files) 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/B is initiating minor compaction (all files) 2024-12-08T01:17:07,661 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/A in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:07,661 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/B in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:07,661 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e841aac9d1ef4d9988fa9b89e33fdf48, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e503788aa62f4ccf9b098bcda1a4891a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b6b733fcb03c47dca3737c7b25ae0a1d] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=36.6 K 2024-12-08T01:17:07,661 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/58357a097b1b4bbfa83c102ebfcdd9c5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=92.1 K 2024-12-08T01:17:07,661 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. files: [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/58357a097b1b4bbfa83c102ebfcdd9c5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb] 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting e841aac9d1ef4d9988fa9b89e33fdf48, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620621526 2024-12-08T01:17:07,661 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58357a097b1b4bbfa83c102ebfcdd9c5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620621526 2024-12-08T01:17:07,662 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting e503788aa62f4ccf9b098bcda1a4891a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733620622700 2024-12-08T01:17:07,662 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31567d7faf5e4e4b9a1a8a19abd8f41e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733620622700 2024-12-08T01:17:07,662 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b733fcb03c47dca3737c7b25ae0a1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733620624938 2024-12-08T01:17:07,662 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f5f4f8e94d4581b01394c8188ae5bb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733620624938 2024-12-08T01:17:07,666 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:07,667 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208ecd6f4617a5e4f4c8ec5fa41b4381f6b_375eeb2eab877f849984ffb3528951c4 store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:07,668 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#B#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:17:07,668 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/a9ce47d0ccde4e9b807f66e39f2d8da5 is 50, key is test_row_0/B:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:07,670 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208ecd6f4617a5e4f4c8ec5fa41b4381f6b_375eeb2eab877f849984ffb3528951c4, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:07,670 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ecd6f4617a5e4f4c8ec5fa41b4381f6b_375eeb2eab877f849984ffb3528951c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:07,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742425_1601 (size=4469) 2024-12-08T01:17:07,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742424_1600 (size=12983) 2024-12-08T01:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T01:17:07,682 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:17:08,079 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:17:08,079 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/833bd071b5454c4c81bf92353cfa34da is 175, key is test_row_0/A:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:08,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742426_1602 (size=31937) 2024-12-08T01:17:08,083 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/a9ce47d0ccde4e9b807f66e39f2d8da5 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a9ce47d0ccde4e9b807f66e39f2d8da5 2024-12-08T01:17:08,089 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/B of 375eeb2eab877f849984ffb3528951c4 into a9ce47d0ccde4e9b807f66e39f2d8da5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:17:08,089 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:08,089 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/B, priority=13, startTime=1733620627660; duration=0sec 2024-12-08T01:17:08,089 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T01:17:08,089 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:B 2024-12-08T01:17:08,089 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T01:17:08,090 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T01:17:08,090 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1541): 375eeb2eab877f849984ffb3528951c4/C is initiating minor compaction (all files) 2024-12-08T01:17:08,090 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 375eeb2eab877f849984ffb3528951c4/C in TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:08,090 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/beb7ff5aef4e46b095a1e4c8a22964d1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/df4d11dfaea44f70a4bd42f82e1dfaba, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2bcf277411184882b51bfdfdb865136f] into tmpdir=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp, totalSize=36.6 K 2024-12-08T01:17:08,091 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting beb7ff5aef4e46b095a1e4c8a22964d1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733620621526 2024-12-08T01:17:08,091 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting df4d11dfaea44f70a4bd42f82e1dfaba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733620622700 2024-12-08T01:17:08,092 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bcf277411184882b51bfdfdb865136f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733620624938 2024-12-08T01:17:08,098 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 375eeb2eab877f849984ffb3528951c4#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:17:08,098 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2d2c357c02ce4b4a90899ddc742fed03 is 50, key is test_row_0/C:col10/1733620624938/Put/seqid=0 2024-12-08T01:17:08,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742427_1603 (size=12983) 2024-12-08T01:17:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44843 {}] regionserver.HRegion(8855): Flush requested on 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:08,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T01:17:08,167 INFO [Thread-2338 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2338. 2024-12-08T01:17:08,167 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-12-08T01:17:08,167 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-12-08T01:17:08,167 DEBUG [Thread-2338 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:08,167 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:08,167 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:17:08,167 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:08,167 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:08,167 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:08,167 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:08,167 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:08,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:17:08,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:08,167 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:17:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:08,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:08,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d8cbc548e2504b2a80477f63fc96ffe0_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_0/A:col10/1733620628163/Put/seqid=0 2024-12-08T01:17:08,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742428_1604 (size=12454) 2024-12-08T01:17:08,487 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/833bd071b5454c4c81bf92353cfa34da as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/833bd071b5454c4c81bf92353cfa34da 2024-12-08T01:17:08,491 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/A of 375eeb2eab877f849984ffb3528951c4 into 833bd071b5454c4c81bf92353cfa34da(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:17:08,491 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:08,491 INFO [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/A, priority=13, startTime=1733620627660; duration=0sec 2024-12-08T01:17:08,491 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:08,491 DEBUG [RS:0;96982154c400:44843-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:A 2024-12-08T01:17:08,506 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/2d2c357c02ce4b4a90899ddc742fed03 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2d2c357c02ce4b4a90899ddc742fed03 2024-12-08T01:17:08,509 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 375eeb2eab877f849984ffb3528951c4/C of 375eeb2eab877f849984ffb3528951c4 into 2d2c357c02ce4b4a90899ddc742fed03(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:17:08,509 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:08,509 INFO [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., storeName=375eeb2eab877f849984ffb3528951c4/C, priority=13, startTime=1733620627660; duration=0sec 2024-12-08T01:17:08,509 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T01:17:08,510 DEBUG [RS:0;96982154c400:44843-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 375eeb2eab877f849984ffb3528951c4:C 2024-12-08T01:17:08,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:08,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d8cbc548e2504b2a80477f63fc96ffe0_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d8cbc548e2504b2a80477f63fc96ffe0_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:08,581 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34865a4f8d8b480295335ba5df8d693d, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:08,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34865a4f8d8b480295335ba5df8d693d is 175, key is test_row_0/A:col10/1733620628163/Put/seqid=0 2024-12-08T01:17:08,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742429_1605 (size=31255) 2024-12-08T01:17:08,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34865a4f8d8b480295335ba5df8d693d 2024-12-08T01:17:08,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/6e5c8606505f4b2ab68ec614411f3b7a is 50, key is test_row_0/B:col10/1733620628163/Put/seqid=0 2024-12-08T01:17:09,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742430_1606 (size=12301) 2024-12-08T01:17:09,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/6e5c8606505f4b2ab68ec614411f3b7a 2024-12-08T01:17:09,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/d8f8a32c11044b0b8acd742644d2550d is 50, key is test_row_0/C:col10/1733620628163/Put/seqid=0 2024-12-08T01:17:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742431_1607 (size=12301) 2024-12-08T01:17:09,123 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-12-08T01:17:09,123 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:09,123 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:09,123 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:09,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:09,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/d8f8a32c11044b0b8acd742644d2550d 2024-12-08T01:17:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/34865a4f8d8b480295335ba5df8d693d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34865a4f8d8b480295335ba5df8d693d 2024-12-08T01:17:09,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34865a4f8d8b480295335ba5df8d693d, entries=150, sequenceid=317, filesize=30.5 K 2024-12-08T01:17:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/6e5c8606505f4b2ab68ec614411f3b7a as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/6e5c8606505f4b2ab68ec614411f3b7a 2024-12-08T01:17:09,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/6e5c8606505f4b2ab68ec614411f3b7a, entries=150, sequenceid=317, filesize=12.0 K 2024-12-08T01:17:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/d8f8a32c11044b0b8acd742644d2550d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d8f8a32c11044b0b8acd742644d2550d 2024-12-08T01:17:09,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d8f8a32c11044b0b8acd742644d2550d, entries=150, sequenceid=317, filesize=12.0 K 2024-12-08T01:17:09,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for 375eeb2eab877f849984ffb3528951c4 in 1270ms, sequenceid=317, compaction requested=false 2024-12-08T01:17:09,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 375eeb2eab877f849984ffb3528951c4: 2024-12-08T01:17:16,086 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-12-08T01:17:16,086 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T01:17:16,086 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:16,087 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:16,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4831 2024-12-08T01:17:16,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4533 2024-12-08T01:17:16,089 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4577 2024-12-08T01:17:16,089 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4814 2024-12-08T01:17:16,089 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4544 2024-12-08T01:17:16,089 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T01:17:16,089 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:17:16,090 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:17:16,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:16,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:16,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:16,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T01:17:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T01:17:16,095 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620636094"}]},"ts":"1733620636094"} 2024-12-08T01:17:16,096 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T01:17:16,096 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T01:17:16,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T01:17:16,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, UNASSIGN}] 2024-12-08T01:17:16,098 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, UNASSIGN 2024-12-08T01:17:16,098 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=CLOSING, regionLocation=96982154c400,44843,1733620436630 2024-12-08T01:17:16,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, UNASSIGN because future has completed 2024-12-08T01:17:16,099 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:17:16,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630}] 2024-12-08T01:17:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T01:17:16,253 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:16,253 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:17:16,253 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 375eeb2eab877f849984ffb3528951c4, disabling compactions & flushes 2024-12-08T01:17:16,253 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:16,253 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:16,253 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. after waiting 0 ms 2024-12-08T01:17:16,253 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:16,253 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 375eeb2eab877f849984ffb3528951c4 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-08T01:17:16,254 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=A 2024-12-08T01:17:16,254 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:16,254 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=B 2024-12-08T01:17:16,254 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:16,254 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 375eeb2eab877f849984ffb3528951c4, store=C 2024-12-08T01:17:16,254 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T01:17:16,264 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dab43bf52fa94fb387a778fdfabaa5d2_375eeb2eab877f849984ffb3528951c4 is 50, key is test_row_1/A:col10/1733620636083/Put/seqid=0 2024-12-08T01:17:16,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742432_1608 (size=7374) 2024-12-08T01:17:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T01:17:16,667 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:17:16,673 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dab43bf52fa94fb387a778fdfabaa5d2_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dab43bf52fa94fb387a778fdfabaa5d2_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:16,675 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/1899fa52f75448158384e0cc126fb322, store: [table=TestAcidGuarantees family=A region=375eeb2eab877f849984ffb3528951c4] 2024-12-08T01:17:16,676 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/1899fa52f75448158384e0cc126fb322 is 175, key is test_row_1/A:col10/1733620636083/Put/seqid=0 2024-12-08T01:17:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742433_1609 (size=13865) 2024-12-08T01:17:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T01:17:17,081 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/1899fa52f75448158384e0cc126fb322 2024-12-08T01:17:17,095 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/317c9ba8ec1948e8a9f424c24419dc56 is 50, key is test_row_1/B:col10/1733620636083/Put/seqid=0 2024-12-08T01:17:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742434_1610 (size=7415) 2024-12-08T01:17:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T01:17:17,502 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/317c9ba8ec1948e8a9f424c24419dc56 2024-12-08T01:17:17,514 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/c4341e7f206443f0a63239acf97e2ed2 is 50, key is test_row_1/C:col10/1733620636083/Put/seqid=0 2024-12-08T01:17:17,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742435_1611 (size=7415) 2024-12-08T01:17:17,918 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/c4341e7f206443f0a63239acf97e2ed2 2024-12-08T01:17:17,923 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/A/1899fa52f75448158384e0cc126fb322 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/1899fa52f75448158384e0cc126fb322 2024-12-08T01:17:17,928 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/1899fa52f75448158384e0cc126fb322, entries=50, sequenceid=324, filesize=13.5 K 2024-12-08T01:17:17,929 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/B/317c9ba8ec1948e8a9f424c24419dc56 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/317c9ba8ec1948e8a9f424c24419dc56 2024-12-08T01:17:17,934 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/317c9ba8ec1948e8a9f424c24419dc56, entries=50, sequenceid=324, filesize=7.2 K 2024-12-08T01:17:17,936 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/.tmp/C/c4341e7f206443f0a63239acf97e2ed2 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/c4341e7f206443f0a63239acf97e2ed2 2024-12-08T01:17:17,941 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/c4341e7f206443f0a63239acf97e2ed2, entries=50, sequenceid=324, filesize=7.2 K 2024-12-08T01:17:17,942 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 375eeb2eab877f849984ffb3528951c4 in 1689ms, sequenceid=324, compaction requested=true 2024-12-08T01:17:17,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3c545eff9d8e48f79f553cc0b47dc004, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/814dc4defb6a4954895434044e734dd1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/4304e0263bfc472b9c3867881ff13984, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/856f8351d63043bc8451b2724a088650, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/58357a097b1b4bbfa83c102ebfcdd9c5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb] to archive 2024-12-08T01:17:17,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:17:17,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/7ea3ba98b9804c5cbab7886c81fc1441 2024-12-08T01:17:17,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/a673a6eb9e9448c292475ce9e05770f1 2024-12-08T01:17:17,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/8dcc1a1459824369acf5bb0658a5ada3 2024-12-08T01:17:17,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3c545eff9d8e48f79f553cc0b47dc004 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3c545eff9d8e48f79f553cc0b47dc004 2024-12-08T01:17:17,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/317c8c558d5f442eb687a84171744bcb 2024-12-08T01:17:17,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/731c719012ad43ecbea7e872747305b2 2024-12-08T01:17:17,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/814dc4defb6a4954895434044e734dd1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/814dc4defb6a4954895434044e734dd1 2024-12-08T01:17:17,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/bc386961d7ba4116a8d5d55a8cc38afc 2024-12-08T01:17:17,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34c46289f2664f62abe5e298a467f67c 2024-12-08T01:17:17,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/4304e0263bfc472b9c3867881ff13984 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/4304e0263bfc472b9c3867881ff13984 2024-12-08T01:17:17,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/dabb163d23a24d099fb6e6fa3230c73a 2024-12-08T01:17:17,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/11393aff1e7d449d8a8dc918534cca03 2024-12-08T01:17:17,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/856f8351d63043bc8451b2724a088650 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/856f8351d63043bc8451b2724a088650 2024-12-08T01:17:17,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/0c85bc21af904b02b68598892182b222 2024-12-08T01:17:17,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/381e4af9f1804085a8a26a30b946057c 2024-12-08T01:17:17,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/51ce8858bf584a9281055d24b677e820 2024-12-08T01:17:17,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/58357a097b1b4bbfa83c102ebfcdd9c5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/58357a097b1b4bbfa83c102ebfcdd9c5 2024-12-08T01:17:17,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/3eebfbcb55b24be8a5977f75511367ab 2024-12-08T01:17:17,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/31567d7faf5e4e4b9a1a8a19abd8f41e 2024-12-08T01:17:17,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/79f5f4f8e94d4581b01394c8188ae5bb 2024-12-08T01:17:17,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b29b003cf4e04cb1ae4afd8dbd7df51b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/bfc3ddb88ed4406dbf2a3600f72044e0, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ffc3b5366a1a4b2986dd2b3781bd237d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/38700f0469d2479996af1cc7d1515815, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/80b16635ccf64bfe9b138543a70d6680, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7b81f6b224244bc99730707efe39b3a3, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b77c9354bf784a79bf653001086d742a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/c26554abc3e84eb7b44cf0022c4cd838, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a127abe0643745e2b7d6ffe9cf91d679, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/efee0627f2c24ec8b682a45d4fecae08, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7d009fa70dc047e2bdf291a2f86ddc4f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/0a4190e15e5a424697be68ceb55ce9d9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/de96d659ef3d49d2ab358360564afc78, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/3af6aabeda294969880eb1efb3404b0a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/431971fa339248e684bd236d5602ee31, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ec51e41361be49de83d1d96614bcc985, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e841aac9d1ef4d9988fa9b89e33fdf48, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/4010cd83cc0e46e2a682bc4a9fba373f, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e503788aa62f4ccf9b098bcda1a4891a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b6b733fcb03c47dca3737c7b25ae0a1d] to archive 2024-12-08T01:17:17,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:17:17,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b29b003cf4e04cb1ae4afd8dbd7df51b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b29b003cf4e04cb1ae4afd8dbd7df51b 2024-12-08T01:17:17,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/bfc3ddb88ed4406dbf2a3600f72044e0 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/bfc3ddb88ed4406dbf2a3600f72044e0 2024-12-08T01:17:17,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ffc3b5366a1a4b2986dd2b3781bd237d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ffc3b5366a1a4b2986dd2b3781bd237d 2024-12-08T01:17:17,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/38700f0469d2479996af1cc7d1515815 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/38700f0469d2479996af1cc7d1515815 2024-12-08T01:17:17,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/80b16635ccf64bfe9b138543a70d6680 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/80b16635ccf64bfe9b138543a70d6680 2024-12-08T01:17:17,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7b81f6b224244bc99730707efe39b3a3 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7b81f6b224244bc99730707efe39b3a3 2024-12-08T01:17:17,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b77c9354bf784a79bf653001086d742a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b77c9354bf784a79bf653001086d742a 2024-12-08T01:17:17,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/c26554abc3e84eb7b44cf0022c4cd838 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/c26554abc3e84eb7b44cf0022c4cd838 2024-12-08T01:17:17,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a127abe0643745e2b7d6ffe9cf91d679 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a127abe0643745e2b7d6ffe9cf91d679 2024-12-08T01:17:17,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/efee0627f2c24ec8b682a45d4fecae08 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/efee0627f2c24ec8b682a45d4fecae08 2024-12-08T01:17:17,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7d009fa70dc047e2bdf291a2f86ddc4f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/7d009fa70dc047e2bdf291a2f86ddc4f 2024-12-08T01:17:17,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/0a4190e15e5a424697be68ceb55ce9d9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/0a4190e15e5a424697be68ceb55ce9d9 2024-12-08T01:17:17,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/de96d659ef3d49d2ab358360564afc78 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/de96d659ef3d49d2ab358360564afc78 2024-12-08T01:17:17,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/3af6aabeda294969880eb1efb3404b0a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/3af6aabeda294969880eb1efb3404b0a 2024-12-08T01:17:17,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/431971fa339248e684bd236d5602ee31 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/431971fa339248e684bd236d5602ee31 2024-12-08T01:17:17,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ec51e41361be49de83d1d96614bcc985 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/ec51e41361be49de83d1d96614bcc985 2024-12-08T01:17:17,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e841aac9d1ef4d9988fa9b89e33fdf48 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e841aac9d1ef4d9988fa9b89e33fdf48 2024-12-08T01:17:17,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/4010cd83cc0e46e2a682bc4a9fba373f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/4010cd83cc0e46e2a682bc4a9fba373f 2024-12-08T01:17:17,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e503788aa62f4ccf9b098bcda1a4891a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/e503788aa62f4ccf9b098bcda1a4891a 2024-12-08T01:17:17,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b6b733fcb03c47dca3737c7b25ae0a1d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/b6b733fcb03c47dca3737c7b25ae0a1d 2024-12-08T01:17:17,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d3b82b087c49407e908ae2c36d641e8d, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/1358d78667b1448b9d8b75dc661d4241, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/8f7914826f4a45ecac446bf063fc34b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07e16b00e151414680849d55a4cee8cd, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07937c3054db4d12b19dcc62b3d3baf6, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/97d49721e8c644f189294cfff3ea886b, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/7e5f4d05350144de8d3f5c8ff0ffb8b5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2f50dd1123c1461dac206bda32cbf6d7, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/37174bf6563d488b991d311e44b8d4c9, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9699795173484a098bd67871bd3d409a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9403426e2847468dbbe75a3db4ed15fc, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9d978fb31afd4bc7bc8dd6561fc9d3fb, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/ed3e51db2ef2437fb463c8058cbaf7e2, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/40187649a4744676a892837ebfb7c331, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/fbfaf519e7a546fba2943d62c8fde458, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/f13d3efcbce445ea92c4a1ea35e3f54a, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/beb7ff5aef4e46b095a1e4c8a22964d1, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/edd5c037e3f04c4d8d0c16154ec8cfc5, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/df4d11dfaea44f70a4bd42f82e1dfaba, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2bcf277411184882b51bfdfdb865136f] to archive 2024-12-08T01:17:17,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:17:17,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d3b82b087c49407e908ae2c36d641e8d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d3b82b087c49407e908ae2c36d641e8d 2024-12-08T01:17:17,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/1358d78667b1448b9d8b75dc661d4241 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/1358d78667b1448b9d8b75dc661d4241 2024-12-08T01:17:17,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/8f7914826f4a45ecac446bf063fc34b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/8f7914826f4a45ecac446bf063fc34b5 2024-12-08T01:17:17,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07e16b00e151414680849d55a4cee8cd to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07e16b00e151414680849d55a4cee8cd 2024-12-08T01:17:17,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07937c3054db4d12b19dcc62b3d3baf6 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/07937c3054db4d12b19dcc62b3d3baf6 2024-12-08T01:17:17,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/97d49721e8c644f189294cfff3ea886b to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/97d49721e8c644f189294cfff3ea886b 2024-12-08T01:17:17,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/7e5f4d05350144de8d3f5c8ff0ffb8b5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/7e5f4d05350144de8d3f5c8ff0ffb8b5 2024-12-08T01:17:17,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2f50dd1123c1461dac206bda32cbf6d7 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2f50dd1123c1461dac206bda32cbf6d7 2024-12-08T01:17:17,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/37174bf6563d488b991d311e44b8d4c9 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/37174bf6563d488b991d311e44b8d4c9 2024-12-08T01:17:17,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9699795173484a098bd67871bd3d409a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9699795173484a098bd67871bd3d409a 2024-12-08T01:17:17,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9403426e2847468dbbe75a3db4ed15fc to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9403426e2847468dbbe75a3db4ed15fc 2024-12-08T01:17:17,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9d978fb31afd4bc7bc8dd6561fc9d3fb to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/9d978fb31afd4bc7bc8dd6561fc9d3fb 2024-12-08T01:17:17,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/ed3e51db2ef2437fb463c8058cbaf7e2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/ed3e51db2ef2437fb463c8058cbaf7e2 2024-12-08T01:17:17,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/40187649a4744676a892837ebfb7c331 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/40187649a4744676a892837ebfb7c331 2024-12-08T01:17:17,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/fbfaf519e7a546fba2943d62c8fde458 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/fbfaf519e7a546fba2943d62c8fde458 2024-12-08T01:17:17,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/f13d3efcbce445ea92c4a1ea35e3f54a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/f13d3efcbce445ea92c4a1ea35e3f54a 2024-12-08T01:17:17,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/beb7ff5aef4e46b095a1e4c8a22964d1 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/beb7ff5aef4e46b095a1e4c8a22964d1 2024-12-08T01:17:17,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/edd5c037e3f04c4d8d0c16154ec8cfc5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/edd5c037e3f04c4d8d0c16154ec8cfc5 2024-12-08T01:17:17,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/df4d11dfaea44f70a4bd42f82e1dfaba to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/df4d11dfaea44f70a4bd42f82e1dfaba 2024-12-08T01:17:17,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2bcf277411184882b51bfdfdb865136f to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2bcf277411184882b51bfdfdb865136f 2024-12-08T01:17:17,999 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/recovered.edits/327.seqid, newMaxSeqId=327, maxSeqId=4 2024-12-08T01:17:18,000 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. 2024-12-08T01:17:18,000 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 375eeb2eab877f849984ffb3528951c4: Waiting for close lock at 1733620636253Running coprocessor pre-close hooks at 1733620636253Disabling compacts and flushes for region at 1733620636253Disabling writes for close at 1733620636253Obtaining lock to block concurrent updates at 1733620636254 (+1 ms)Preparing flush snapshotting stores in 375eeb2eab877f849984ffb3528951c4 at 1733620636254Finished memstore snapshotting TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733620636254Flushing stores of TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4. at 1733620636256 (+2 ms)Flushing 375eeb2eab877f849984ffb3528951c4/A: creating writer at 1733620636256Flushing mob file 375eeb2eab877f849984ffb3528951c4/A: appending metadata at 1733620636264 (+8 ms)Flushing mob file 375eeb2eab877f849984ffb3528951c4/A: closing flushed file at 1733620636264Flushing 375eeb2eab877f849984ffb3528951c4/A: appending metadata at 1733620636675 (+411 ms)Flushing 375eeb2eab877f849984ffb3528951c4/A: closing flushed file at 1733620636675Flushing 375eeb2eab877f849984ffb3528951c4/B: creating writer at 1733620637090 (+415 ms)Flushing 375eeb2eab877f849984ffb3528951c4/B: appending metadata at 1733620637095 (+5 ms)Flushing 375eeb2eab877f849984ffb3528951c4/B: closing flushed file at 1733620637095Flushing 375eeb2eab877f849984ffb3528951c4/C: creating writer at 1733620637511 (+416 ms)Flushing 375eeb2eab877f849984ffb3528951c4/C: appending metadata at 1733620637514 (+3 ms)Flushing 375eeb2eab877f849984ffb3528951c4/C: closing flushed file at 1733620637514Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@280f3f2d: reopening flushed file at 1733620637923 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27be7fad: reopening flushed file at 1733620637928 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@423648ec: reopening flushed file at 1733620637935 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 375eeb2eab877f849984ffb3528951c4 in 1689ms, sequenceid=324, compaction requested=true at 1733620637942 (+7 ms)Writing region close event to WAL at 1733620637997 (+55 ms)Running coprocessor post-close hooks at 1733620638000 (+3 ms)Closed at 1733620638000 2024-12-08T01:17:18,001 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,002 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=375eeb2eab877f849984ffb3528951c4, regionState=CLOSED 2024-12-08T01:17:18,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 because future has completed 2024-12-08T01:17:18,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-08T01:17:18,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 375eeb2eab877f849984ffb3528951c4, server=96982154c400,44843,1733620436630 in 1.9040 sec 2024-12-08T01:17:18,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-08T01:17:18,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=375eeb2eab877f849984ffb3528951c4, UNASSIGN in 1.9070 sec 2024-12-08T01:17:18,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-08T01:17:18,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9100 sec 2024-12-08T01:17:18,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620638007"}]},"ts":"1733620638007"} 2024-12-08T01:17:18,008 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T01:17:18,008 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T01:17:18,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9180 sec 2024-12-08T01:17:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T01:17:18,243 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:17:18,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T01:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:18,248 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T01:17:18,250 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:18,256 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,258 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C, FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/recovered.edits] 2024-12-08T01:17:18,261 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/1899fa52f75448158384e0cc126fb322 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/1899fa52f75448158384e0cc126fb322 2024-12-08T01:17:18,263 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34865a4f8d8b480295335ba5df8d693d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/34865a4f8d8b480295335ba5df8d693d 2024-12-08T01:17:18,264 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/833bd071b5454c4c81bf92353cfa34da to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/A/833bd071b5454c4c81bf92353cfa34da 2024-12-08T01:17:18,267 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/317c9ba8ec1948e8a9f424c24419dc56 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/317c9ba8ec1948e8a9f424c24419dc56 2024-12-08T01:17:18,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/6e5c8606505f4b2ab68ec614411f3b7a to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/6e5c8606505f4b2ab68ec614411f3b7a 2024-12-08T01:17:18,271 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a9ce47d0ccde4e9b807f66e39f2d8da5 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/B/a9ce47d0ccde4e9b807f66e39f2d8da5 2024-12-08T01:17:18,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2d2c357c02ce4b4a90899ddc742fed03 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/2d2c357c02ce4b4a90899ddc742fed03 2024-12-08T01:17:18,276 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/c4341e7f206443f0a63239acf97e2ed2 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/c4341e7f206443f0a63239acf97e2ed2 2024-12-08T01:17:18,278 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d8f8a32c11044b0b8acd742644d2550d to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/C/d8f8a32c11044b0b8acd742644d2550d 2024-12-08T01:17:18,282 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/recovered.edits/327.seqid to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4/recovered.edits/327.seqid 2024-12-08T01:17:18,283 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/default/TestAcidGuarantees/375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,283 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T01:17:18,284 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T01:17:18,285 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T01:17:18,290 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208045d2842b25d49c4a04433cad904b4ae_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208045d2842b25d49c4a04433cad904b4ae_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,292 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080c23bbded9a54352aa60d61c37f84c90_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080c23bbded9a54352aa60d61c37f84c90_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,294 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120813f72ea03679439ba18d242ea3c1db52_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120813f72ea03679439ba18d242ea3c1db52_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,297 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081a095c226d4f4b538b8a65dedb3f383c_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081a095c226d4f4b538b8a65dedb3f383c_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,299 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082fcbebb938c9475081c71ec7a5e0b8a1_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082fcbebb938c9475081c71ec7a5e0b8a1_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,302 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084376b0ba87fa4851863b5cca939f501d_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084376b0ba87fa4851863b5cca939f501d_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,304 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208607908f0dbb84489818a35af25b7fea6_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208607908f0dbb84489818a35af25b7fea6_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,306 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086bb48c01e1f647a1b09453fb774feef0_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086bb48c01e1f647a1b09453fb774feef0_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,308 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120885d3a87e64e143f480149635d806ea08_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120885d3a87e64e143f480149635d806ea08_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,309 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088f561677f78c4493ab54815f394dde46_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088f561677f78c4493ab54815f394dde46_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089956ea4ef3f84dec922a804887ef854f_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089956ea4ef3f84dec922a804887ef854f_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089b13c5f17a0148eea72ff599411f71ba_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089b13c5f17a0148eea72ff599411f71ba_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,312 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bae6dc7534e846928ea1b4139f51d345_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bae6dc7534e846928ea1b4139f51d345_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,313 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d76736ea37ce43139b4b4a729283bfcc_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d76736ea37ce43139b4b4a729283bfcc_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d8cbc548e2504b2a80477f63fc96ffe0_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d8cbc548e2504b2a80477f63fc96ffe0_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dab43bf52fa94fb387a778fdfabaa5d2_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dab43bf52fa94fb387a778fdfabaa5d2_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe144d9539bd4f7f9d64764d34040131_375eeb2eab877f849984ffb3528951c4 to hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe144d9539bd4f7f9d64764d34040131_375eeb2eab877f849984ffb3528951c4 2024-12-08T01:17:18,316 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T01:17:18,317 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:18,319 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T01:17:18,321 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T01:17:18,322 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:18,322 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T01:17:18,322 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733620638322"}]},"ts":"9223372036854775807"} 2024-12-08T01:17:18,323 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T01:17:18,323 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 375eeb2eab877f849984ffb3528951c4, NAME => 'TestAcidGuarantees,,1733620603709.375eeb2eab877f849984ffb3528951c4.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T01:17:18,323 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T01:17:18,324 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733620638323"}]},"ts":"9223372036854775807"} 2024-12-08T01:17:18,325 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T01:17:18,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T01:17:18,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-12-08T01:17:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T01:17:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T01:17:18,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T01:17:18,362 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 248), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 474), ProcessCount=11 (was 11), AvailableMemoryMB=9705 (was 10176) 2024-12-08T01:17:18,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T01:17:18,362 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:17:18,362 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:17:18,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:18,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:18,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:17:18,362 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T01:17:18,362 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-12-08T01:17:18,363 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=96982154c400,37013,1733620435769 2024-12-08T01:17:18,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:17:18,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:17:18,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:17:18,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:17:18,432 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T01:17:18,433 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:17:18,434 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:17:18,434 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:17:18,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:18,434 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:17:18,435 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,44843,1733620436630' ***** 2024-12-08T01:17:18,435 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T01:17:18,435 INFO [RS:0;96982154c400:44843 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:17:18,436 INFO [RS:0;96982154c400:44843 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T01:17:18,436 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:17:18,436 INFO [RS:0;96982154c400:44843 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T01:17:18,436 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(959): stopping server 96982154c400,44843,1733620436630 2024-12-08T01:17:18,436 INFO [RS:0;96982154c400:44843 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:17:18,437 INFO [RS:0;96982154c400:44843 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;96982154c400:44843. 2024-12-08T01:17:18,437 DEBUG [RS:0;96982154c400:44843 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:17:18,437 DEBUG [RS:0;96982154c400:44843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:18,437 INFO [RS:0;96982154c400:44843 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:17:18,437 INFO [RS:0;96982154c400:44843 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:17:18,437 INFO [RS:0;96982154c400:44843 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:17:18,437 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T01:17:18,438 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T01:17:18,438 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T01:17:18,438 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:18,438 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:17:18,438 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:17:18,438 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:17:18,438 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:17:18,438 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:17:18,438 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-08T01:17:18,456 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/info/011571da10e0406f8fa7e5ad1ede727d is 95, key is TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446./info:/1733620474961/DeleteFamily/seqid=0 2024-12-08T01:17:18,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742436_1612 (size=6514) 2024-12-08T01:17:18,495 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:17:18,514 INFO [regionserver/96982154c400:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T01:17:18,514 INFO [regionserver/96982154c400:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T01:17:18,638 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:18,839 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:18,860 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/info/011571da10e0406f8fa7e5ad1ede727d 2024-12-08T01:17:18,885 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/ns/e75425f6cd5e4b45a98f4172fa216c33 is 93, key is TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446./ns:/1733620474919/DeleteFamily/seqid=0 2024-12-08T01:17:18,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742437_1613 (size=5906) 2024-12-08T01:17:19,039 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:19,239 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:19,290 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/ns/e75425f6cd5e4b45a98f4172fa216c33 2024-12-08T01:17:19,314 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/rep_barrier/438bd101655e454eac3d6e5132984ac6 is 102, key is TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446./rep_barrier:/1733620474919/DeleteFamily/seqid=0 2024-12-08T01:17:19,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742438_1614 (size=6025) 2024-12-08T01:17:19,439 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:19,640 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:19,719 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/rep_barrier/438bd101655e454eac3d6e5132984ac6 2024-12-08T01:17:19,744 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/table/bd0dd617533340988759251758a0ec94 is 96, key is TestAcidGuarantees,,1733620441137.13ab931b3a97ca4195bbab8a55597446./table:/1733620474919/DeleteFamily/seqid=0 2024-12-08T01:17:19,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742439_1615 (size=5936) 2024-12-08T01:17:19,840 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:20,040 DEBUG [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:17:20,148 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/table/bd0dd617533340988759251758a0ec94 2024-12-08T01:17:20,151 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/info/011571da10e0406f8fa7e5ad1ede727d as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/info/011571da10e0406f8fa7e5ad1ede727d 2024-12-08T01:17:20,153 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/info/011571da10e0406f8fa7e5ad1ede727d, entries=12, sequenceid=90, filesize=6.4 K 2024-12-08T01:17:20,154 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/ns/e75425f6cd5e4b45a98f4172fa216c33 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/ns/e75425f6cd5e4b45a98f4172fa216c33 2024-12-08T01:17:20,157 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/ns/e75425f6cd5e4b45a98f4172fa216c33, entries=8, sequenceid=90, filesize=5.8 K 2024-12-08T01:17:20,158 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/rep_barrier/438bd101655e454eac3d6e5132984ac6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/rep_barrier/438bd101655e454eac3d6e5132984ac6 2024-12-08T01:17:20,161 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/rep_barrier/438bd101655e454eac3d6e5132984ac6, entries=6, sequenceid=90, filesize=5.9 K 2024-12-08T01:17:20,162 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/.tmp/table/bd0dd617533340988759251758a0ec94 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/table/bd0dd617533340988759251758a0ec94 2024-12-08T01:17:20,164 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/table/bd0dd617533340988759251758a0ec94, entries=7, sequenceid=90, filesize=5.8 K 2024-12-08T01:17:20,167 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false 2024-12-08T01:17:20,177 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-08T01:17:20,177 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:17:20,178 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:17:20,178 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620638438Running coprocessor pre-close hooks at 1733620638438Disabling compacts and flushes for region at 1733620638438Disabling writes for close at 1733620638438Obtaining lock to block concurrent updates at 1733620638438Preparing flush snapshotting stores in 1588230740 at 1733620638438Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733620638439 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733620638439Flushing 1588230740/info: creating writer at 1733620638440 (+1 ms)Flushing 1588230740/info: appending metadata at 1733620638455 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733620638456 (+1 ms)Flushing 1588230740/ns: creating writer at 1733620638870 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733620638885 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733620638885Flushing 1588230740/rep_barrier: creating writer at 1733620639301 (+416 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733620639314 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733620639314Flushing 1588230740/table: creating writer at 1733620639728 (+414 ms)Flushing 1588230740/table: appending metadata at 1733620639744 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733620639744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a729cea: reopening flushed file at 1733620640150 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62c7e25: reopening flushed file at 1733620640153 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2afd586f: reopening flushed file at 1733620640157 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@644dee75: reopening flushed file at 1733620640161 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false at 1733620640167 (+6 ms)Writing region close event to WAL at 1733620640174 (+7 ms)Running coprocessor post-close hooks at 1733620640177 (+3 ms)Closed at 1733620640177 2024-12-08T01:17:20,178 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T01:17:20,241 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(976): stopping server 96982154c400,44843,1733620436630; all regions closed. 2024-12-08T01:17:20,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741834_1010 (size=25344) 2024-12-08T01:17:20,248 DEBUG [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/oldWALs 2024-12-08T01:17:20,248 INFO [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C44843%2C1733620436630.meta:.meta(num 1733620440682) 2024-12-08T01:17:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741832_1008 (size=17266422) 2024-12-08T01:17:20,252 DEBUG [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/oldWALs 2024-12-08T01:17:20,252 INFO [RS:0;96982154c400:44843 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C44843%2C1733620436630:(num 1733620439656) 2024-12-08T01:17:20,252 DEBUG [RS:0;96982154c400:44843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:17:20,252 INFO [RS:0;96982154c400:44843 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:17:20,252 INFO [RS:0;96982154c400:44843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:17:20,253 INFO [RS:0;96982154c400:44843 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T01:17:20,253 INFO [RS:0;96982154c400:44843 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:17:20,253 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:17:20,254 INFO [RS:0;96982154c400:44843 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44843 2024-12-08T01:17:20,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:17:20,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,44843,1733620436630 2024-12-08T01:17:20,298 INFO [RS:0;96982154c400:44843 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:17:20,298 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,44843,1733620436630] 2024-12-08T01:17:20,318 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,44843,1733620436630 already deleted, retry=false 2024-12-08T01:17:20,319 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 96982154c400,44843,1733620436630 expired; onlineServers=0 2024-12-08T01:17:20,319 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '96982154c400,37013,1733620435769' ***** 2024-12-08T01:17:20,319 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T01:17:20,319 INFO [M:0;96982154c400:37013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:17:20,319 INFO [M:0;96982154c400:37013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:17:20,319 DEBUG [M:0;96982154c400:37013 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T01:17:20,320 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T01:17:20,320 DEBUG [M:0;96982154c400:37013 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T01:17:20,320 DEBUG [master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620439382 {}] cleaner.HFileCleaner(306): Exit Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620439382,5,FailOnTimeoutGroup] 2024-12-08T01:17:20,320 DEBUG [master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620439383 {}] cleaner.HFileCleaner(306): Exit Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620439383,5,FailOnTimeoutGroup] 2024-12-08T01:17:20,320 INFO [M:0;96982154c400:37013 {}] hbase.ChoreService(370): Chore service for: master/96982154c400:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T01:17:20,320 INFO [M:0;96982154c400:37013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:17:20,320 DEBUG [M:0;96982154c400:37013 {}] master.HMaster(1795): Stopping service threads 2024-12-08T01:17:20,320 INFO [M:0;96982154c400:37013 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T01:17:20,320 INFO [M:0;96982154c400:37013 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T01:17:20,321 ERROR [M:0;96982154c400:37013 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:41355 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41355,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-08T01:17:20,321 INFO [M:0;96982154c400:37013 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T01:17:20,322 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T01:17:20,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T01:17:20,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:17:20,330 DEBUG [M:0;96982154c400:37013 {}] zookeeper.ZKUtil(347): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T01:17:20,330 WARN [M:0;96982154c400:37013 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T01:17:20,330 INFO [M:0;96982154c400:37013 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/.lastflushedseqids 2024-12-08T01:17:20,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742440_1616 (size=72) 2024-12-08T01:17:20,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:17:20,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44843-0x10002fc8ee80001, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:17:20,409 INFO [RS:0;96982154c400:44843 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:17:20,409 INFO [RS:0;96982154c400:44843 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,44843,1733620436630; zookeeper connection closed. 2024-12-08T01:17:20,409 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@459943b8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@459943b8 2024-12-08T01:17:20,410 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T01:17:20,744 INFO [M:0;96982154c400:37013 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T01:17:20,744 INFO [M:0;96982154c400:37013 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T01:17:20,744 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T01:17:20,745 INFO [M:0;96982154c400:37013 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:17:20,745 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:17:20,745 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T01:17:20,745 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:17:20,745 INFO [M:0;96982154c400:37013 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=744.89 KB heapSize=910.20 KB 2024-12-08T01:17:20,763 DEBUG [M:0;96982154c400:37013 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b4d580868afe4670bbda96b7088751b1 is 82, key is hbase:meta,,1/info:regioninfo/1733620440757/Put/seqid=0 2024-12-08T01:17:20,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742441_1617 (size=5672) 2024-12-08T01:17:21,167 INFO [M:0;96982154c400:37013 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2034 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b4d580868afe4670bbda96b7088751b1 2024-12-08T01:17:21,195 DEBUG [M:0;96982154c400:37013 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9141482de3794c81a6ee5f2ea9a22dc6 is 2479, key is \x00\x00\x00\x00\x00\x00\x00'/proc:d/1733620478134/Put/seqid=0 2024-12-08T01:17:21,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742442_1618 (size=44650) 2024-12-08T01:17:21,601 INFO [M:0;96982154c400:37013 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=744.33 KB at sequenceid=2034 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9141482de3794c81a6ee5f2ea9a22dc6 2024-12-08T01:17:21,609 INFO [M:0;96982154c400:37013 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9141482de3794c81a6ee5f2ea9a22dc6 2024-12-08T01:17:21,627 DEBUG [M:0;96982154c400:37013 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7139544fe19142af892038a545ac0cf4 is 69, key is 96982154c400,44843,1733620436630/rs:state/1733620439421/Put/seqid=0 2024-12-08T01:17:21,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073742443_1619 (size=5156) 2024-12-08T01:17:22,031 INFO [M:0;96982154c400:37013 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2034 (bloomFilter=true), to=hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7139544fe19142af892038a545ac0cf4 2024-12-08T01:17:22,034 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b4d580868afe4670bbda96b7088751b1 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b4d580868afe4670bbda96b7088751b1 2024-12-08T01:17:22,037 INFO [M:0;96982154c400:37013 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b4d580868afe4670bbda96b7088751b1, entries=8, sequenceid=2034, filesize=5.5 K 2024-12-08T01:17:22,038 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9141482de3794c81a6ee5f2ea9a22dc6 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9141482de3794c81a6ee5f2ea9a22dc6 2024-12-08T01:17:22,042 INFO [M:0;96982154c400:37013 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9141482de3794c81a6ee5f2ea9a22dc6 2024-12-08T01:17:22,042 INFO [M:0;96982154c400:37013 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9141482de3794c81a6ee5f2ea9a22dc6, entries=166, sequenceid=2034, filesize=43.6 K 2024-12-08T01:17:22,043 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7139544fe19142af892038a545ac0cf4 as hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7139544fe19142af892038a545ac0cf4 2024-12-08T01:17:22,046 INFO [M:0;96982154c400:37013 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41355/user/jenkins/test-data/789b3ea7-b720-bf5e-e8a3-d1d697cc2720/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7139544fe19142af892038a545ac0cf4, entries=1, sequenceid=2034, filesize=5.0 K 2024-12-08T01:17:22,047 INFO [M:0;96982154c400:37013 {}] regionserver.HRegion(3140): Finished flush of dataSize ~744.89 KB/762764, heapSize ~909.90 KB/931736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1302ms, sequenceid=2034, compaction requested=false 2024-12-08T01:17:22,048 INFO [M:0;96982154c400:37013 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:17:22,049 DEBUG [M:0;96982154c400:37013 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733620640744Disabling compacts and flushes for region at 1733620640744Disabling writes for close at 1733620640745 (+1 ms)Obtaining lock to block concurrent updates at 1733620640745Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733620640745Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=762764, getHeapSize=931976, getOffHeapSize=0, getCellsCount=2225 at 1733620640746 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733620640746Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733620640747 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733620640763 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733620640763Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733620641170 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733620641195 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733620641195Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733620641609 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733620641627 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733620641627Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40cdd04c: reopening flushed file at 1733620642034 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@537e933: reopening flushed file at 1733620642038 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6363dfee: reopening flushed file at 1733620642042 (+4 ms)Finished flush of dataSize ~744.89 KB/762764, heapSize ~909.90 KB/931736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1302ms, sequenceid=2034, compaction requested=false at 1733620642047 (+5 ms)Writing region close event to WAL at 1733620642048 (+1 ms)Closed at 1733620642048 2024-12-08T01:17:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40511 is added to blk_1073741830_1006 (size=896157) 2024-12-08T01:17:22,053 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:17:22,054 INFO [M:0;96982154c400:37013 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T01:17:22,054 INFO [M:0;96982154c400:37013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37013 2024-12-08T01:17:22,054 INFO [M:0;96982154c400:37013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:17:22,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:17:22,219 INFO [M:0;96982154c400:37013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:17:22,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37013-0x10002fc8ee80000, quorum=127.0.0.1:63353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:17:22,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:17:22,233 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:17:22,233 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:17:22,234 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:17:22,234 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/hadoop.log.dir/,STOPPED} 2024-12-08T01:17:22,236 WARN [BP-261549399-172.17.0.2-1733620432050 heartbeating to localhost/127.0.0.1:41355 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:17:22,236 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:17:22,236 WARN [BP-261549399-172.17.0.2-1733620432050 heartbeating to localhost/127.0.0.1:41355 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-261549399-172.17.0.2-1733620432050 (Datanode Uuid 6c6b5ef4-4866-4f22-9083-19ac04e152bb) service to localhost/127.0.0.1:41355 2024-12-08T01:17:22,236 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:17:22,238 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/data/data1/current/BP-261549399-172.17.0.2-1733620432050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:17:22,239 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/cluster_7a803019-5c06-ba8d-1bd8-5a3d94590269/data/data2/current/BP-261549399-172.17.0.2-1733620432050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:17:22,239 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:17:22,246 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T01:17:22,247 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:17:22,247 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:17:22,247 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:17:22,247 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7b0d9b4-dc90-373c-396e-5896d6ff68d8/hadoop.log.dir/,STOPPED} 2024-12-08T01:17:22,260 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T01:17:22,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down